Add API10 system tests for downstream response body limits (#7046)

uurien · cursoragent · web-flow · commit b76eee535b5a · 2026-06-23T15:15:24.000+02:00
Co-authored-by: Cursor &lt;cursoragent@cursor.com&gt;
diff --git a/docs/understand/weblogs/end-to-end_weblog.md b/docs/understand/weblogs/end-to-end_weblog.md
@@ -225,6 +225,20 @@ if the request to `internal_server` is a failure, it must return a json body wit
 - `status` the status code of the `internal_server` response if available or a null value
 - `error` a string describing the error, for debug purposes
 
+### GET /external_request/body_limit/{failure_reason}
+### POST /external_request/body_limit/{failure_reason}
+### TRACE /external_request/body_limit/{failure_reason}
+### PUT /external_request/body_limit/{failure_reason}
+
+Same behavior as `/external_request`, but the downstream call targets `http://internal_server:8089/downstream_response/{failure_reason}`.
+
+Supported `{failure_reason}` values (defined in `internal_server`):
+- `invalid_content_type`
+- `content_length_missing`
+- `content_length_too_big`
+
+Unknown values must return HTTP 404 with a json error body.
+
 ### GET /external_request/redirect
 
 This endpoint tests HTTP redirect chains with downstream requests, using the fastapi application in `/utils/build/docker/internal_server/app.py`
diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml
@@ -238,6 +238,9 @@ manifest:
   tests/appsec/rasp/test_api10.py::Test_API10_request_headers: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_request_method: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_body: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_headers: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_status: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_without_downstream_body_analysis_using_max: missing_feature
diff --git a/manifests/golang.yml b/manifests/golang.yml
@@ -214,6 +214,9 @@ manifest:
   tests/appsec/rasp/test_api10.py::Test_API10_request_headers: v2.7.0-dev
   tests/appsec/rasp/test_api10.py::Test_API10_request_method: v2.4.0
   tests/appsec/rasp/test_api10.py::Test_API10_response_body: v2.7.0-dev
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_headers: v2.4.0
   tests/appsec/rasp/test_api10.py::Test_API10_response_status: v2.4.0
   tests/appsec/rasp/test_api10.py::Test_API10_without_downstream_body_analysis_using_max: v2.4.0
diff --git a/manifests/java.yml b/manifests/java.yml
@@ -1205,6 +1205,9 @@ manifest:
     - weblog_declaration:
         "*": missing_feature
         vertx3: v1.60.1  # TODO: a lower version might be supported
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_headers:
     - weblog_declaration:
         "*": missing_feature
diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml
@@ -97,6 +97,7 @@ refs:
   - &ref_5_104_0 '>=5.104.0'
   - &ref_5_106_0 '>=5.106.0'
   - &ref_5_107_0 '>=5.107.0'
+  - &ref_5_109_0 '>=5.109.0'
   - &ref_5_110_0 '>=5.110.0'
   - &ref_6_0_0 '>=6.0.0-pre'
 manifest:
@@ -706,6 +707,9 @@ manifest:
   tests/appsec/rasp/test_api10.py::Test_API10_request_headers: *ref_5_87_0
   tests/appsec/rasp/test_api10.py::Test_API10_request_method: *ref_5_87_0
   tests/appsec/rasp/test_api10.py::Test_API10_response_body: *ref_5_87_0
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: *ref_5_109_0
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: *ref_5_109_0
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: *ref_5_109_0
   tests/appsec/rasp/test_api10.py::Test_API10_response_headers: *ref_5_87_0
   tests/appsec/rasp/test_api10.py::Test_API10_response_status: *ref_5_87_0
   tests/appsec/rasp/test_api10.py::Test_API10_without_downstream_body_analysis_using_max: *ref_5_87_0
diff --git a/manifests/php.yml b/manifests/php.yml
@@ -227,6 +227,9 @@ manifest:
   tests/appsec/rasp/test_api10.py::Test_API10_request_headers: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_request_method: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_body: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_headers: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_status: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_without_downstream_body_analysis_using_max: missing_feature
diff --git a/manifests/python.yml b/manifests/python.yml
@@ -586,6 +586,9 @@ manifest:
         fastapi: v3.15.0.dev (with requests support)
         *django: v4.3.0-dev0 (with httpx support)
   tests/appsec/rasp/test_api10.py::Test_API10_response_body: v4.5.0-rc1
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_headers:
     - weblog_declaration:
         "*": v3.14.0.rc
diff --git a/manifests/ruby.yml b/manifests/ruby.yml
@@ -393,6 +393,9 @@ manifest:
     - weblog_declaration:
         "*": v2.29.0
         "graphql23": irrelevant
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: missing_feature
+  tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: missing_feature
   tests/appsec/rasp/test_api10.py::Test_API10_response_headers:
     - weblog_declaration:
         "*": v2.27.0-dev
diff --git a/tests/appsec/rasp/test_api10.py b/tests/appsec/rasp/test_api10.py
@@ -339,6 +339,78 @@ def test_api10_res_body(self):
         interfaces.library.validate_one_span(self.r, validator=self.validate_absence)
 
 
+class API10ResponseBodyIgnored(API10):
+    """Base for downstream response body guard tests."""
+
+    TAGS_NOT_EXPECTED = ["_dd.appsec.trace.res_body"]
+
+    def validate_ignored(self, span: DataDogLibrarySpan):
+        if span.get("parent_id") not in (0, None):
+            return None
+
+        if not self.validate_metric(span):
+            return None
+
+        for tag in self.TAGS_NOT_EXPECTED:
+            assert tag not in span.get("meta", {}), f"Tag {tag} should NOT be present in span's meta"
+
+        return True
+
+
+@rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx")
+@features.api10
+@scenarios.appsec_rasp
+class Test_API10_response_body_ignored_content_type(API10ResponseBodyIgnored):
+    """Downstream response body ignored when content-type is unsupported."""
+
+    TAGS_EXPECTED_METRIC = [
+        ("_dd.appsec.downstream_request.response_body_ignored.content_type_invalid", "1"),
+    ]
+
+    def setup_api10_response_body_ignored_content_type(self):
+        self.r = weblog.get("/external_request/body_limit/invalid_content_type")
+
+    def test_api10_response_body_ignored_content_type(self):
+        assert self.r.status_code == 200
+        interfaces.library.validate_one_span(self.r, validator=self.validate_ignored)
+
+
+@rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx")
+@features.api10
+@scenarios.appsec_rasp
+class Test_API10_response_body_ignored_content_length_missing(API10ResponseBodyIgnored):
+    """Downstream response body ignored when content-length is missing."""
+
+    TAGS_EXPECTED_METRIC = [
+        ("_dd.appsec.downstream_request.response_body_ignored.content_length_missing", "1"),
+    ]
+
+    def setup_api10_response_body_ignored_content_length_missing(self):
+        self.r = weblog.get("/external_request/body_limit/content_length_missing")
+
+    def test_api10_response_body_ignored_content_length_missing(self):
+        assert self.r.status_code == 200
+        interfaces.library.validate_one_span(self.r, validator=self.validate_ignored)
+
+
+@rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx")
+@features.api10
+@scenarios.appsec_rasp
+class Test_API10_response_body_ignored_content_length_too_big(API10ResponseBodyIgnored):
+    """Downstream response body ignored when content-length exceeds maxBytes."""
+
+    TAGS_EXPECTED_METRIC = [
+        ("_dd.appsec.downstream_request.response_body_ignored.content_length_too_big", "1"),
+    ]
+
+    def setup_api10_response_body_ignored_content_length_too_big(self):
+        self.r = weblog.get("/external_request/body_limit/content_length_too_big")
+
+    def test_api10_response_body_ignored_content_length_too_big(self):
+        assert self.r.status_code == 200
+        interfaces.library.validate_one_span(self.r, validator=self.validate_ignored)
+
+
 @rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx")
 @features.api10
 @scenarios.appsec_rasp
diff --git a/utils/_context/_scenarios/appsec_rasp.py b/utils/_context/_scenarios/appsec_rasp.py
@@ -24,6 +24,7 @@ def __init__(
             # added to test Test_ExtendedRequestBodyCollection
             "DD_APPSEC_RASP_COLLECT_REQUEST_BODY": "true",
             "DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE": "1.0",
+            "DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES": "128",
             "OPENAI_BASE_URL": "http://internal_server:8089",
         }
         merged_env = default_env | weblog_env
diff --git a/utils/build/docker/internal_server/app.py b/utils/build/docker/internal_server/app.py
@@ -40,6 +40,45 @@ async def mirror(status: int, request: fastapi.Request):
     return fastapi.responses.JSONResponse({"status": "OK", "payload": body}, status_code=status, headers=query)
 
 
+_DOWNSTREAM_RESPONSE_BODY = b'{"ok":true}'
+_DOWNSTREAM_RESPONSE_BODY_TOO_BIG = b'{"payload":"' + b"x" * 180 + b'"}'
+
+
+@app.get("/downstream_response/{profile}")
+async def downstream_response(profile: str):
+    """Controlled downstream responses for API10 response body limit guards."""
+
+    if profile == "invalid_content_type":
+        return fastapi.responses.Response(
+            content=_DOWNSTREAM_RESPONSE_BODY,
+            status_code=200,
+            media_type="text/html",
+            headers={"content-length": str(len(_DOWNSTREAM_RESPONSE_BODY))},
+        )
+
+    if profile == "content_length_missing":
+
+        async def body_stream():
+            yield _DOWNSTREAM_RESPONSE_BODY
+
+        return fastapi.responses.StreamingResponse(
+            body_stream(),
+            status_code=200,
+            media_type="application/json",
+        )
+
+    if profile == "content_length_too_big":
+        body = _DOWNSTREAM_RESPONSE_BODY_TOO_BIG
+        return fastapi.responses.Response(
+            content=body,
+            status_code=200,
+            media_type="application/json",
+            headers={"content-length": str(len(body))},
+        )
+
+    return fastapi.responses.JSONResponse({"error": "unknown profile"}, status_code=404)
+
+
 @app.get("/redirect", response_class=fastapi.responses.RedirectResponse)
 async def redirect(request: fastapi.Request):
     """Redirect endpoint for testing API 10 with redirects
diff --git a/utils/build/docker/nodejs/express/app.js b/utils/build/docker/nodejs/express/app.js
@@ -664,12 +664,20 @@ app.get('/add_event', (req, res) => {
   res.status(200).json({ message: 'Event added' })
 })
 
-app.all('/external_request', (req, res) => {
+const DOWNSTREAM_RESPONSE_BODY_LIMIT_PROFILES = new Set([
+  'invalid_content_type',
+  'content_length_missing',
+  'content_length_too_big'
+])
+
+function forwardExternalRequest (req, res, downstreamPath) {
   const status = req.query.status || '200'
   const urlExtra = req.query.url_extra || ''
 
   const headers = {}
+  const queryParamsExcludedFromHeaders = new Set(['status', 'url_extra'])
   for (const [key, value] of Object.entries(req.query)) {
+    if (queryParamsExcludedFromHeaders.has(key)) continue
     headers[key] = String(value)
   }
 
@@ -679,10 +687,12 @@ app.all('/external_request', (req, res) => {
     headers['Content-Type'] = req.headers['content-type'] || 'application/json'
   }
 
+  const path = downstreamPath || `/mirror/${status}${urlExtra}`
+
   const options = {
     hostname: 'internal_server',
     port: 8089,
-    path: `/mirror/${status}${urlExtra}`,
+    path,
     method: req.method,
     headers
   }
@@ -703,12 +713,25 @@ app.all('/external_request', (req, res) => {
     })
   })
 
-  // Write body if present
   if (body) {
     request.write(body)
   }
 
   request.end()
+}
+
+app.all('/external_request', (req, res) => {
+  forwardExternalRequest(req, res)
+})
+
+app.all('/external_request/body_limit/:failureReason', (req, res) => {
+  const { failureReason } = req.params
+  if (!DOWNSTREAM_RESPONSE_BODY_LIMIT_PROFILES.has(failureReason)) {
+    res.status(404).json({ error: 'unknown failure reason' })
+    return
+  }
+
+  forwardExternalRequest(req, res, `/downstream_response/${failureReason}`)
 })
 
 app.get('/external_request/redirect', (req, res) => {
diff --git a/utils/build/docker/nodejs/express4-typescript/app.ts b/utils/build/docker/nodejs/express4-typescript/app.ts
@@ -412,12 +412,20 @@ app.get('/set_cookie', (req: Request, res: Response) => {
   res.send('OK')
 })
 
-app.all('/external_request', (req: Request, res: Response) => {
+const DOWNSTREAM_RESPONSE_BODY_LIMIT_PROFILES = new Set([
+  'invalid_content_type',
+  'content_length_missing',
+  'content_length_too_big'
+])
+
+function forwardExternalRequest (req: Request, res: Response, downstreamPath?: string) {
   const status = req.query.status || '200'
   const urlExtra = req.query.url_extra || ''
 
   const headers: any = {}
+  const queryParamsExcludedFromHeaders = new Set(['status', 'url_extra'])
   for (const [key, value] of Object.entries(req.query)) {
+    if (queryParamsExcludedFromHeaders.has(key)) continue
     headers[key] = String(value)
   }
 
@@ -427,10 +435,12 @@ app.all('/external_request', (req: Request, res: Response) => {
     headers['Content-Type'] = req.headers['content-type'] || 'application/json'
   }
 
+  const path = downstreamPath || `/mirror/${status}${urlExtra}`
+
   const options = {
     hostname: 'internal_server',
     port: 8089,
-    path: `/mirror/${status}${urlExtra}`,
+    path,
     method: req.method,
     headers
   }
@@ -451,12 +461,25 @@ app.all('/external_request', (req: Request, res: Response) => {
     })
   })
 
-  // Write body if present
   if (body) {
     request.write(body)
   }
 
   request.end()
+}
+
+app.all('/external_request', (req: Request, res: Response) => {
+  forwardExternalRequest(req, res)
+})
+
+app.all('/external_request/body_limit/:failureReason', (req: Request, res: Response) => {
+  const { failureReason } = req.params
+  if (!DOWNSTREAM_RESPONSE_BODY_LIMIT_PROFILES.has(failureReason)) {
+    res.status(404).json({ error: 'unknown failure reason' })
+    return
+  }
+
+  forwardExternalRequest(req, res, `/downstream_response/${failureReason}`)
 })
 
 app.get('/external_request/redirect', (req: Request, res: Response) => {
diff --git a/utils/build/docker/nodejs/fastify/app.js b/utils/build/docker/nodejs/fastify/app.js
diff --git a/utils/build/docker/nodejs/nextjs/src/app/external_request/body_limit/[failureReason]/route.js b/utils/build/docker/nodejs/nextjs/src/app/external_request/body_limit/[failureReason]/route.js
diff --git a/utils/build/docker/nodejs/nextjs/src/app/external_request/forwardToInternalServer.js b/utils/build/docker/nodejs/nextjs/src/app/external_request/forwardToInternalServer.js
diff --git a/utils/build/docker/nodejs/nextjs/src/app/external_request/route.js b/utils/build/docker/nodejs/nextjs/src/app/external_request/route.js
diff --git a/utils/telemetry/intake/static/config_norm_rules.json b/utils/telemetry/intake/static/config_norm_rules.json

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ def __init__(`
`24`	`24`	`# added to test Test_ExtendedRequestBodyCollection`
`25`	`25`	`"DD_APPSEC_RASP_COLLECT_REQUEST_BODY": "true",`
`26`	`26`	`"DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE": "1.0",`
	`27`	`+ "DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES": "128",`
`27`	`28`	`"OPENAI_BASE_URL": "http://internal_server:8089",`
`28`	`29`	`}`
`29`	`30`	`merged_env = default_env \| weblog_env`