Add a new end point that allows the SPA recommendation to be called without the shard parameter (#3251)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -42828,6 +42828,9 @@ components:
       description: Attributes of the SPA Recommendation resource. Contains recommendations
         for both driver and executor components.
       properties:
+        confidence_level:
+          format: double
+          type: number
         driver:
           $ref: '#/components/schemas/ComponentRecommendation'
         executor:
@@ -86626,13 +86629,79 @@ paths:
       x-unstable: '**Note**: This feature is in private beta. To request access, use
         the request access form in the [Service Level Objectives](https://docs.datadoghq.com/service_management/service_level_objectives/#slo-csv-export)
         docs.'
-  /api/v2/spa/recommendations/{service}/{shard}:
+  /api/v2/spa/recommendations/{service}:
     get:
-      description: Retrieve resource recommendations for a Spark job. The caller (Spark
-        Gateway or DJM UI) provides a service name and shard identifier, and SPA returns
-        structured recommendations for driver and executor resources.
+      description: This endpoint is currently experimental and restricted to Datadog
+        internal use only. Retrieve resource recommendations for a Spark job. The
+        caller (Spark Gateway or DJM UI) provides a service name and SPA returns structured
+        recommendations for driver and executor resources. The version with a shard
+        should be preferred, where possible, as it gives more accurate results.
       operationId: GetSPARecommendations
       parameters:
+      - description: The recommendation service should not use its metrics cache.
+        in: query
+        name: bypass_cache
+        schema:
+          type: string
+      - description: The service name for a spark job.
+        in: path
+        name: service
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/json:
+              example:
+                data:
+                  attributes:
+                    driver:
+                      estimation:
+                        cpu:
+                          max: 1500
+                          p75: 1000
+                          p95: 1200
+                        ephemeral_storage: 896
+                        heap: 6144
+                        memory: 7168
+                        overhead: 1024
+                    executor:
+                      estimation:
+                        cpu:
+                          max: 2000
+                          p75: 1200
+                          p95: 1500
+                        ephemeral_storage: 512
+                        heap: 3072
+                        memory: 4096
+                        overhead: 1024
+                  id: dedupeactivecontexts:adp_dedupeactivecontexts_org2
+                  type: recommendation
+              schema:
+                $ref: '#/components/schemas/RecommendationDocument'
+          description: OK
+        '400':
+          $ref: '#/components/responses/BadRequestResponse'
+        '403':
+          $ref: '#/components/responses/NotAuthorizedResponse'
+        '429':
+          $ref: '#/components/responses/TooManyRequestsResponse'
+      security:
+      - AuthZ: []
+      summary: Get SPA Recommendations
+      tags:
+      - Spa
+      x-unstable: '**Note**: This endpoint is in preview and may change in the future.
+        It is not yet recommended for production use.'
+  /api/v2/spa/recommendations/{service}/{shard}:
+    get:
+      description: This endpoint is currently experimental and restricted to Datadog
+        internal use only. Retrieve resource recommendations for a Spark job. The
+        caller (Spark Gateway or DJM UI) provides a service name and shard identifier,
+        and SPA returns structured recommendations for driver and executor resources.
+      operationId: GetSPARecommendationsWithShard
+      parameters:
       - description: The shard tag for a spark job, which differentiates jobs within
           the same service that have different resource needs
         in: path
@@ -86646,6 +86715,11 @@ paths:
         required: true
         schema:
           type: string
+      - description: The recommendation service should not use its metrics cache.
+        in: query
+        name: bypass_cache
+        schema:
+          type: string
       responses:
         '200':
           content:
@@ -86684,11 +86758,13 @@ paths:
           $ref: '#/components/responses/NotAuthorizedResponse'
         '429':
           $ref: '#/components/responses/TooManyRequestsResponse'
-      summary: Get SPA Recommendations
+      security:
+      - AuthZ: []
+      summary: Get SPA Recommendations with a shard parameter
       tags:
       - Spa
-      x-unstable: '**Note**: This endpoint is in public beta and may change in the
-        future. It is not yet recommended for production use.'
+      x-unstable: '**Note**: This endpoint is in preview and may change in the future.
+        It is not yet recommended for production use.'
   /api/v2/spans/analytics/aggregate:
     post:
       description: 'The API endpoint to aggregate spans into buckets and compute metrics

features/v2/spa.feature

@@ -4,29 +4,49 @@ Feature: Spa
   insights to help optimize Spark job configurations.
 
   Background:
-    Given a valid "apiKeyAuth" key in the system
-    And a valid "appKeyAuth" key in the system
-    And an instance of "Spa" API
-    And operation "GetSPARecommendations" enabled
-    And new "GetSPARecommendations" request
+    Given an instance of "Spa" API
 
   @generated @skip @team:DataDog/data-and-analytics-processing
   Scenario: Get SPA Recommendations returns "Bad Request" response
-    Given request contains "shard" parameter from "REPLACE.ME"
+    Given operation "GetSPARecommendations" enabled
+    And new "GetSPARecommendations" request
     And request contains "service" parameter from "REPLACE.ME"
     When the request is sent
     Then the response status is 400 Bad Request
 
   @generated @skip @team:DataDog/data-and-analytics-processing
   Scenario: Get SPA Recommendations returns "OK" response
-    Given request contains "shard" parameter from "REPLACE.ME"
+    Given operation "GetSPARecommendations" enabled
+    And new "GetSPARecommendations" request
+    And request contains "service" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 200 OK
+
+  @generated @skip @team:DataDog/data-and-analytics-processing
+  Scenario: Get SPA Recommendations with a shard parameter returns "Bad Request" response
+    Given operation "GetSPARecommendationsWithShard" enabled
+    And new "GetSPARecommendationsWithShard" request
+    And request contains "shard" parameter from "REPLACE.ME"
+    And request contains "service" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/data-and-analytics-processing
+  Scenario: Get SPA Recommendations with a shard parameter returns "OK" response
+    Given operation "GetSPARecommendationsWithShard" enabled
+    And new "GetSPARecommendationsWithShard" request
+    And request contains "shard" parameter from "REPLACE.ME"
     And request contains "service" parameter from "REPLACE.ME"
     When the request is sent
     Then the response status is 200 OK
 
   @skip @team:DataDog/data-and-analytics-processing
   Scenario: GetSPARecommendations returns a JSON:API Recommendation with driver and executor estimations
-    Given request contains "service" parameter with value "dedupeactivecontexts"
+    Given new "GetSPARecommendations" request
+    And a valid "apiKeyAuth" key in the system
+    And a valid "appKeyAuth" key in the system
+    And operation "GetSPARecommendations" enabled
+    And request contains "service" parameter with value "dedupeactivecontexts"
     And request contains "shard" parameter with value "adp_dedupeactivecontexts_org2"
     When the request is sent
     Then the response status is 404 Not Found

features/v2/undo.json

@@ -4484,6 +4484,12 @@
       "type": "safe"
     }
   },
+  "GetSPARecommendationsWithShard": {
+    "tag": "Spa",
+    "undo": {
+      "type": "safe"
+    }
+  },
   "AggregateSpans": {
     "tag": "Spans",
     "undo": {

private/bdd_runner/src/support/scenarios_model_mapping.ts

@@ -9071,6 +9071,17 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = {
     operationResponseType: "SLOReportStatusGetResponse",
   },
   "SpaApi.V2.GetSPARecommendations": {
+    bypassCache: {
+      type: "string",
+      format: "",
+    },
+    service: {
+      type: "string",
+      format: "",
+    },
+    operationResponseType: "RecommendationDocument",
+  },
+  "SpaApi.V2.GetSPARecommendationsWithShard": {
     shard: {
       type: "string",
       format: "",
@@ -9079,6 +9090,10 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = {
       type: "string",
       format: "",
     },
+    bypassCache: {
+      type: "string",
+      format: "",
+    },
     operationResponseType: "RecommendationDocument",
   },
   "SpansApi.V2.AggregateSpans": {