[AAP] API10 downstream request analysis (#8232)

## Summary of changes
Implement needed changes to handle API10 (downstream request)
vulnerability in RASP

## Reason for change

## Implementation details

## Test coverage

## Other details
<!-- Fixes #{issue} -->


<!--  ⚠️ Note:

Where possible, please obtain 2 approvals prior to merging. Unless
CODEOWNERS specifies otherwise, for external teams it is typically best
to have one review from a team member, and one review from apm-dotnet.
Trivial changes do not require 2 reviews.

MergeQueue is NOT enabled in this repository. If you have write access
to the repo, the PR has 1-2 approvals (see above), and all of the
required checks have passed, you can use the Squash and Merge button to
merge the PR. If you don't have write access, or you need help, reach
out in the #apm-dotnet channel in Slack.
-->

---------

Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com>
Co-authored-by: Andrew Lock <andrewlock.net@gmail.com>

Author: daniel-romano-DD

tracer/missing-nullability-files.csv

@@ -222,6 +222,8 @@ src/Datadog.Trace/Agent/Transports/HttpStreamRequest.cs
 src/Datadog.Trace/Agent/Transports/HttpStreamRequestFactory.cs
 src/Datadog.Trace/Agent/Transports/MimeTypes.cs
 src/Datadog.Trace/Agent/Transports/SocketHandlerRequestFactory.cs
+src/Datadog.Trace/AppSec/Rasp/DownstreamSampler.cs
+src/Datadog.Trace/AppSec/Rasp/IDownstreamSampler.cs
 src/Datadog.Trace/AppSec/Waf/WafConstants.cs
 src/Datadog.Trace/AppSec/Waf/WafReturnCode.cs
 src/Datadog.Trace/ClrProfiler/Helpers/Interception.cs

tracer/src/Datadog.Trace/AppSec/AddressesConstants.cs

@@ -14,7 +14,6 @@ internal static class AddressesConstants
         public const string FileAccess = "server.io.fs.file";
         public const string DBStatement = "server.db.statement";
         public const string DBSystem = "server.db.system";
-        public const string UrlAccess = "server.io.net.url";
         public const string ShellInjection = "server.sys.shell.cmd";
         public const string CommandInjection = "server.sys.exec.cmd";
         public const string RequestMethod = "server.request.method";
@@ -32,6 +31,14 @@ internal static class AddressesConstants
         public const string ResponseBody = "server.response.body";
         public const string ResponseHeaderNoCookies = "server.response.headers.no_cookies";
 
+        public const string DownstreamUrl = "server.io.net.url";
+        public const string DownstreamRequestHeaders = "server.io.net.request.headers";
+        public const string DownstreamRequestMethod = "server.io.net.request.method";
+        public const string DownstreamRequestBody = "server.io.net.request.body";
+        public const string DownstreamResponseStatus = "server.io.net.response.status";
+        public const string DownstreamResponseHeaders = "server.io.net.response.headers";
+        public const string DownstreamResponseBody = "server.io.net.response.body";
+
         public const string UserId = "usr.id";
         public const string UserLogin = "usr.login";
         public const string UserSessionId = "usr.session_id";

tracer/src/Datadog.Trace/AppSec/AppSecRequestContext.cs

@@ -1,12 +1,12 @@
-// <copyright file="AppSecRequestContext.cs" company="Datadog">
+// <copyright file="AppSecRequestContext.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
 
 #nullable enable
 
+using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Threading;
 using Datadog.Trace.AppSec.Rasp;
 using Datadog.Trace.AppSec.Waf;
 using Datadog.Trace.Logging;
@@ -26,6 +26,7 @@ internal sealed partial class AppSecRequestContext
     private readonly object _sync = new();
     private readonly RaspMetricsHelper? _raspMetricsHelper = Security.Instance.RaspEnabled ? new RaspMetricsHelper() : null;
     private readonly List<object> _wafSecurityEvents = new();
+    private readonly ConcurrentDictionary<ulong, bool> _sampledHttpClientRequests = new();
     private int _wafTimeout;
     private int? _wafError;
     private int? _wafRaspError;
@@ -140,6 +141,26 @@ internal void AddStackTrace(string stackCategory, Dictionary<string, object> sta
             _raspStackTraces[stackCategory].Add(stackTrace);
         }
     }
+
+    public bool IsHttpClientRequestSampled(ulong id)
+    {
+        if (_sampledHttpClientRequests.TryGetValue(id, out bool value))
+        {
+            return value;
+        }
+
+        if (Security.Instance.SampleDownstreamRequest(this, id))
+        {
+            if (_sampledHttpClientRequests.Count < Security.Instance.ApiSecurityMaxDownstreamRequestBodyAnalysis)
+            {
+                _sampledHttpClientRequests[id] = true;
+                return true;
+            }
+        }
+
+        _sampledHttpClientRequests[id] = false;
+        return false;
+    }
 }
 
 internal partial class AppSecRequestContext

tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Core.cs

@@ -1,4 +1,4 @@
-// <copyright file="SecurityCoordinator.Core.cs" company="Datadog">
+// <copyright file="SecurityCoordinator.Core.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>

tracer/src/Datadog.Trace/AppSec/Rasp/BodyParser.cs

@@ -0,0 +1,135 @@
+// <copyright file="BodyParser.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Datadog.Trace.Util.Json;
+using Datadog.Trace.Vendors.Newtonsoft.Json;
+
+namespace Datadog.Trace.AppSec.Rasp;
+
+internal static class BodyParser
+{
+    private const int MaxElements = 1000;
+    private const int MaxDepth = 64;
+    private const int MaxStringSize = 1024;
+
+    public static object? Parse(Stream? json)
+    {
+        if (json is null)
+        {
+            return null;
+        }
+
+        using var reader = new StreamReader(json);
+        using var jsonReader = new JsonTextReader(reader) { ArrayPool = JsonArrayPool.Shared };
+        jsonReader.MaxDepth = null; // disable built-in limit; we enforce MaxDepth ourselves
+
+        if (!jsonReader.Read())
+        {
+            return null;
+        }
+
+        try
+        {
+            State state = new(MaxElements);
+            return ReadValue(jsonReader, ref state, 0);
+        }
+        catch (JsonException)
+        {
+            return null;
+        }
+    }
+
+    private static object? ReadValue(JsonTextReader r, ref State state, int depth)
+    {
+        if (depth >= MaxDepth)
+        {
+            state.ObjectTooDeep = true;
+            r.Skip();
+            return null;
+        }
+
+        if (state.ElemsLeft-- <= 0)
+        {
+            state.ListMapTooLarge = true;
+            r.Skip();
+            return null;
+        }
+
+        return r.TokenType switch
+        {
+            JsonToken.StartObject => ReadObject(r, ref state, depth),
+            JsonToken.StartArray => ReadArray(r, ref state, depth),
+            JsonToken.String => ReadString(r, ref state),
+            JsonToken.Integer => r.Value != null ? Convert.ToDouble(r.Value) : null,
+            JsonToken.Float => r.Value != null ? Convert.ToDouble(r.Value) : null,
+            JsonToken.Boolean => r.Value,
+            JsonToken.Null => null,
+            _ => null
+        };
+    }
+
+    private static string? ReadString(JsonTextReader r, ref State state)
+    {
+        string? val = r.Value?.ToString();
+        if (val != null && val.Length > MaxStringSize)
+        {
+            state.StringTooLong = true;
+            val = val.Substring(0, MaxStringSize);
+        }
+
+        return val;
+    }
+
+    private static Dictionary<string, object?> ReadObject(JsonTextReader r, ref State state, int depth)
+    {
+        var dict = new Dictionary<string, object?>();
+
+        while (r.Read() && r.TokenType != JsonToken.EndObject)
+        {
+            if (r.TokenType == JsonToken.PropertyName)
+            {
+                string propertyName = r.Value?.ToString() ?? string.Empty;
+
+                if (r.Read())
+                {
+                    object? val = ReadValue(r, ref state, depth + 1);
+                    if (!state.ListMapTooLarge)
+                    {
+                        dict[propertyName] = val;
+                    }
+                }
+            }
+        }
+
+        return dict;
+    }
+
+    private static List<object?> ReadArray(JsonTextReader r, ref State state, int depth)
+    {
+        var list = new List<object?>();
+
+        while (r.Read() && r.TokenType != JsonToken.EndArray)
+        {
+            object? val = ReadValue(r, ref state, depth + 1);
+            if (!state.ListMapTooLarge)
+            {
+                list.Add(val);
+            }
+        }
+
+        return list;
+    }
+
+    public record struct State(
+        int ElemsLeft = MaxElements,
+        bool ObjectTooDeep = false,
+        bool ListMapTooLarge = false,
+        bool StringTooLong = false);
+}

tracer/src/Datadog.Trace/AppSec/Rasp/DownstreamSampler.cs

@@ -0,0 +1,70 @@
+// <copyright file="DownstreamSampler.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System.Threading;
+
+namespace Datadog.Trace.AppSec.Rasp;
+
+internal sealed class DownstreamSampler : IDownstreamSampler
+{
+    private const long KnuthFactor = 1111111111111111111L;
+    private readonly long _threshold;
+    private long _globalRequestCount;
+
+    public DownstreamSampler(double rate)
+    {
+        double sanitizedRate = rate < 0.0 ? 0 : (rate > 1.0 ? 1 : rate);
+        _threshold = SamplingCutoff(sanitizedRate);
+    }
+
+    private static long SamplingCutoff(double rate)
+    {
+        // Maps a rate in [0.0, 1.0] to a threshold in [long.MinValue, long.MaxValue].
+        // The hashed counter (counter * KnuthFactor) + long.MinValue is uniformly distributed
+        // over all longs, so ~rate fraction of values will be <= the returned threshold.
+        //
+        // ulong.MaxValue as double rounds up to 2^64, so rate * ulong.MaxValue (as double)
+        // can exceed long.MaxValue when rate >= 0.5. To avoid overflow when casting back to long,
+        // subtract long.MinValue (= -2^63) in double arithmetic first, bringing the value into
+        // [0, ~2^63), then cast to long.
+        if (rate >= 1.0)
+        {
+            return long.MaxValue;
+        }
+
+        // rate * 2^64 can exceed long.MaxValue for rate >= 0.5, so we subtract 2^63 first.
+        // long.MinValue == -2^63, so: (long)(rate * 2^64 - 2^63) == (long)(rate * 2^64) + long.MinValue
+        // Both branches are equivalent; using double subtraction first keeps us in range.
+        return (long)((rate * (double)ulong.MaxValue) + long.MinValue);
+    }
+
+    public bool SampleHttpClientRequest(AppSecRequestContext ctx, ulong requestId)
+    {
+        long counter = UpdateRequestCount();
+
+        unchecked
+        {
+            if ((counter * KnuthFactor) + long.MinValue > _threshold)
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private long UpdateRequestCount()
+    {
+        long initial, computed;
+        do
+        {
+            initial = Interlocked.Read(ref _globalRequestCount);
+            computed = (initial == long.MaxValue) ? 0L : initial + 1L;
+        }
+        while (Interlocked.CompareExchange(ref _globalRequestCount, computed, initial) != initial);
+
+        return computed;
+    }
+}

tracer/src/Datadog.Trace/AppSec/Rasp/IDownstreamSampler.cs

@@ -0,0 +1,11 @@
+// <copyright file="IDownstreamSampler.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+namespace Datadog.Trace.AppSec.Rasp;
+
+internal interface IDownstreamSampler
+{
+    bool SampleHttpClientRequest(AppSecRequestContext ctx, ulong requestId);
+}