fix(LaunchDarkly importer): Update API version to 20240415 (#6603)

Co-authored-by: Evandro Myller <22429+emyller@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

api/integrations/launch_darkly/client.py

@@ -1,5 +1,5 @@
 from datetime import timedelta
-from typing import Any, Callable, Generator, Iterator, Optional, TypeVar
+from typing import Any, Callable, Generator, Iterable, Iterator, Optional, TypeVar
 
 import backoff
 from backoff.types import Details
@@ -12,7 +12,9 @@
     BACKOFF_DEFAULT_RETRY_AFTER_SECONDS,
     BACKOFF_MAX_RETRIES,
     LAUNCH_DARKLY_API_BASE_URL,
+    LAUNCH_DARKLY_API_FLAGS_LIMIT_PER_PAGE,
     LAUNCH_DARKLY_API_ITEM_COUNT_LIMIT_PER_PAGE,
+    LAUNCH_DARKLY_API_MAX_ENVIRONMENTS_PER_REQUEST,
     LAUNCH_DARKLY_API_VERSION,
 )
 from integrations.launch_darkly.exceptions import LaunchDarklyRateLimitError
@@ -172,17 +174,46 @@ def get_environments(self, project_key: str) -> list[ld_types.Environment]:
             )
         )
 
-    def get_flags(self, project_key: str) -> list[ld_types.FeatureFlag]:
-        """operationId: getFeatureFlags"""
+    def get_flags_by_envs(
+        self,
+        project_key: str,
+        environment_keys: list[str],
+    ) -> Iterable[ld_types.FeatureFlag]:
+        """
+        Get flags by environment keys.
+
+        :param project_key: Project key to get flags for.
+        :param environment_keys: List of environment keys to include configs for.
+            In API v20240415, the environments field is only returned when filtering.
+            Number of environment keys in one single request is limited to 3.
+        """
         endpoint = f"/api/v2/flags/{project_key}"
-        return list(
-            self._iter_paginated_items(
+        base_params: dict[str, Any] = {
+            "summary": "0",
+            "limit": LAUNCH_DARKLY_API_FLAGS_LIMIT_PER_PAGE,
+        }
+
+        flags_by_key: dict[str, ld_types.FeatureFlag] = {}
+        for i in range(
+            0, len(environment_keys), LAUNCH_DARKLY_API_MAX_ENVIRONMENTS_PER_REQUEST
+        ):
+            batch = environment_keys[
+                i : i + LAUNCH_DARKLY_API_MAX_ENVIRONMENTS_PER_REQUEST
+            ]
+            params = {**base_params, "env": batch}
+
+            flags_iter: Iterator[ld_types.FeatureFlag] = self._iter_paginated_items(
                 collection_endpoint=endpoint,
-                # Summary should be set to 0 in order to get the full flag data including rules.
-                # https://apidocs.launchdarkly.com/tag/Feature-flags#operation/getFeatureFlags!in=query&path=summary&t=request
-                additional_params={"summary": "0"},
+                additional_params=params,
             )
-        )
+            for flag in flags_iter:
+                key = flag["key"]
+                if key in flags_by_key:
+                    flags_by_key[key]["environments"].update(flag["environments"])
+                else:
+                    flags_by_key[key] = flag
+
+        return flags_by_key.values()
 
     def get_flag_count(self, project_key: str) -> int:
         """operationId: getFeatureFlags

api/integrations/launch_darkly/constants.py

@@ -1,8 +1,11 @@
 LAUNCH_DARKLY_API_BASE_URL = "https://app.launchdarkly.com"
-LAUNCH_DARKLY_API_VERSION = "20220603"
-# Maximum limit for /api/v2/projects/
-# /api/v2/flags/ seemingly not limited, but let's not get too greedy
+LAUNCH_DARKLY_API_VERSION = "20240415"
+# Maximum limit for /api/v2/projects/ and other endpoints
 LAUNCH_DARKLY_API_ITEM_COUNT_LIMIT_PER_PAGE = 1000
+# Maximum limit for /api/v2/flags/ is now 100 by launchdarkly API design
+LAUNCH_DARKLY_API_FLAGS_LIMIT_PER_PAGE = 100
+# Maximum limit for env query parameter values in /api/v2/flags/
+LAUNCH_DARKLY_API_MAX_ENVIRONMENTS_PER_REQUEST = 3
 
 LAUNCH_DARKLY_IMPORTED_TAG_COLOR = "#3d4db6"
 LAUNCH_DARKLY_IMPORTED_DEFAULT_TAG_LABEL = "Imported"

api/integrations/launch_darkly/services.py

@@ -1,7 +1,8 @@
+import json
 import logging
 import re
 from contextlib import contextmanager
-from typing import Callable, Generator, Optional, Tuple
+from typing import Any, Callable, Generator, Iterable, Optional, Tuple
 
 from django.conf import settings
 from django.core import signing
@@ -56,6 +57,12 @@ def _unsign_ld_value(value: str, user_id: int) -> str:
     )
 
 
+def _serialize_variation_value(value: Any) -> str:
+    if isinstance(value, (dict, list)):
+        return json.dumps(value)
+    return str(value)
+
+
 def _log_error(
     import_request: LaunchDarklyImportRequest,
     error_message: str,
@@ -710,7 +717,9 @@ def _create_string_feature_states_with_segments_identities(
             enabled_variations = ld_flag_config_summary.get("variations") or {}
             for idx, variation_config in enabled_variations.items():
                 if variation_config.get(variation_config_key):
-                    string_value = variations_by_idx[idx]["value"]
+                    string_value = _serialize_variation_value(
+                        variations_by_idx[idx]["value"]
+                    )
                     break
 
         feature_state, _ = FeatureState.objects.update_or_create(
@@ -758,7 +767,7 @@ def _create_mv_feature_states_with_segments_identities(
 
     for idx, variation in enumerate(variations):
         variation_idx = str(idx)
-        variation_value = variation["value"]
+        variation_value = _serialize_variation_value(variation["value"])
         variation_values_by_idx[variation_idx] = variation_value
         (
             mv_feature_options_by_variation[str(idx)],
@@ -920,7 +929,7 @@ def _create_feature_from_ld(
 
 def _create_features_from_ld(
     import_request: LaunchDarklyImportRequest,
-    ld_flags: list[ld_types.FeatureFlag],
+    ld_flags: Iterable[ld_types.FeatureFlag],
     environments_by_ld_environment_key: dict[str, Environment],
     tags_by_ld_tag: dict[str, Tag],
     segments_by_ld_key: dict[str, Segment],
@@ -1103,7 +1112,10 @@ def process_import_request(
 
         try:
             ld_environments = ld_client.get_environments(project_key=ld_project_key)
-            ld_flags = ld_client.get_flags(project_key=ld_project_key)
+            ld_flags = ld_client.get_flags_by_envs(
+                project_key=ld_project_key,
+                environment_keys=[env["key"] for env in ld_environments],
+            )
             ld_flag_tags = ld_client.get_flag_tags()
             # ld_segment_tags = ld_client.get_segment_tags()
             # Keyed by (segment, environment)

api/tests/unit/integrations/launch_darkly/conftest.py

@@ -31,7 +31,7 @@ def ld_client_mock(
     for method_name, response_data_path in {
         "get_project": "client_responses/get_project.json",
         "get_environments": "client_responses/get_environments.json",
-        "get_flags": "client_responses/get_flags.json",
+        "get_flags_by_envs": "client_responses/get_flags.json",
         "get_segments": "client_responses/get_segments.json",
     }.items():
         getattr(ld_client_mock, method_name).return_value = json.loads(

api/tests/unit/integrations/launch_darkly/test_client.py

@@ -1,4 +1,5 @@
 import json
+from typing import Any
 
 import pytest
 from pytest_mock import MockerFixture
@@ -17,7 +18,7 @@ def test_launch_darkly_client__get_project__return_expected(
     # Given
     token = "test-token"
     project_key = "test-project-key"
-    api_version = "20230101"
+    api_version = "20240415"
 
     mocker.patch(
         "integrations.launch_darkly.client.LAUNCH_DARKLY_API_VERSION",
@@ -53,7 +54,7 @@ def test_launch_darkly_client__get_environments__return_expected(
     # Given
     token = "test-token"
     project_key = "test-project-key"
-    api_version = "20230101"
+    api_version = "20240415"
 
     mocker.patch(
         "integrations.launch_darkly.client.LAUNCH_DARKLY_API_VERSION",
@@ -104,14 +105,14 @@ def test_launch_darkly_client__get_flags__return_expected(
     # Given
     token = "test-token"
     project_key = "test-project-key"
-    api_version = "20230101"
+    api_version = "20240415"
 
     mocker.patch(
         "integrations.launch_darkly.client.LAUNCH_DARKLY_API_VERSION",
         new=api_version,
     )
     mocker.patch(
-        "integrations.launch_darkly.client.LAUNCH_DARKLY_API_ITEM_COUNT_LIMIT_PER_PAGE",
+        "integrations.launch_darkly.client.LAUNCH_DARKLY_API_FLAGS_LIMIT_PER_PAGE",
         new=3,
     )
 
@@ -141,7 +142,12 @@ def test_launch_darkly_client__get_flags__return_expected(
     client = LaunchDarklyClient(token=token)
 
     # When
-    result = client.get_flags(project_key=project_key)
+    result = list(
+        client.get_flags_by_envs(
+            project_key=project_key,
+            environment_keys=["env1", "env2", "env3"],
+        )
+    )
 
     # Then
     assert result == expected_result
@@ -155,7 +161,7 @@ def test_launch_darkly_client__get_flag_count__return_expected(
     # Given
     token = "test-token"
     project_key = "test-project-key"
-    api_version = "20230101"
+    api_version = "20240415"
 
     mocker.patch(
         "integrations.launch_darkly.client.LAUNCH_DARKLY_API_VERSION",
@@ -188,7 +194,7 @@ def test_launch_darkly_client__get_flag_tags__return_expected(
 ) -> None:
     # Given
     token = "test-token"
-    api_version = "20230101"
+    api_version = "20240415"
 
     mocker.patch(
         "integrations.launch_darkly.client.LAUNCH_DARKLY_API_VERSION",
@@ -361,3 +367,115 @@ def test_launch_darkly_client__rate_limit_no_headers__waits_expected(
 
     # Then
     assert result == expected_result
+
+
+def test_launch_darkly_client__get_flags_with_env_keys__return_expected(
+    requests_mock: RequestsMockerFixture,
+) -> None:
+    # Given
+    token = "test-token"
+    project_key = "test-project-env-keys"
+    environment_keys = ["production", "test"]
+
+    response_data = {
+        "items": [
+            {
+                "key": "test-flag",
+                "name": "Test Flag",
+                "kind": "boolean",
+                "environments": {
+                    "production": {"on": True},
+                    "test": {"on": False},
+                },
+            }
+        ],
+        "totalCount": 1,
+    }
+
+    requests_mock.get(
+        f"https://app.launchdarkly.com/api/v2/flags/{project_key}",
+        json=response_data,
+        request_headers={"Authorization": token},
+    )
+
+    client = LaunchDarklyClient(token=token)
+
+    # When
+    result = list(
+        client.get_flags_by_envs(
+            project_key=project_key,
+            environment_keys=environment_keys,
+        )
+    )
+
+    # Then
+    assert result == response_data["items"]
+    assert "env=production" in requests_mock.request_history[0].url
+    assert "env=test" in requests_mock.request_history[0].url
+
+
+def test_launch_darkly_client__get_flags_with_env_keys_batching__merges_environments(
+    requests_mock: RequestsMockerFixture,
+) -> None:
+    # Given
+    token = "test-token"
+    project_key = "test-project-batching"
+    environment_keys = ["env1", "env2", "env3", "env4"]
+
+    batch1_response = {
+        "items": [
+            {
+                "key": "flag1",
+                "environments": {
+                    "env1": {"on": True},
+                    "env2": {"on": False},
+                    "env3": {"on": True},
+                },
+            }
+        ],
+        "totalCount": 1,
+    }
+
+    batch2_response = {
+        "items": [
+            {
+                "key": "flag1",
+                "environments": {
+                    "env4": {"on": False},
+                },
+            }
+        ],
+        "totalCount": 1,
+    }
+
+    requests_mock.get(
+        f"https://app.launchdarkly.com/api/v2/flags/{project_key}",
+        [
+            {"json": batch1_response},
+            {"json": batch2_response},
+        ],
+        request_headers={"Authorization": token},
+    )
+
+    client = LaunchDarklyClient(token=token)
+
+    # When
+    result = list(
+        client.get_flags_by_envs(
+            project_key=project_key,
+            environment_keys=environment_keys,
+        )
+    )
+
+    # Then
+    assert len(result) == 1
+    assert result[0]["key"] == "flag1"
+    environments: Any = result[0]["environments"]
+    assert environments == {
+        "env1": {"on": True},
+        "env2": {"on": False},
+        "env3": {"on": True},
+        "env4": {"on": False},
+    }
+
+    assert len(requests_mock.request_history) == 2

api/tests/unit/integrations/launch_darkly/test_services.py

@@ -17,6 +17,7 @@
 from features.models import Feature, FeatureState
 from integrations.launch_darkly.models import LaunchDarklyImportRequest
 from integrations.launch_darkly.services import (
+    _serialize_variation_value,
     create_import_request,
     process_import_request,
 )
@@ -63,7 +64,8 @@ def test_create_import_request__return_expected(
 
 
 @pytest.mark.parametrize(
-    "failing_ld_client_method_name", ["get_environments", "get_flags", "get_flag_tags"]
+    "failing_ld_client_method_name",
+    ["get_environments", "get_flags_by_envs", "get_flag_tags"],
 )
 @pytest.mark.parametrize(
     "exception, expected_error_message",
@@ -611,3 +613,27 @@ def test_process_import_request__large_segments__correctly_imported(
         ]
     )
     assert buf.getvalue() == expected_condition_data_snapshot
+
+
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        (
+            {"enabled": True, "description": "test"},
+            '{"enabled": true, "description": "test"}',
+        ),
+        ([1, 2, 3], "[1, 2, 3]"),
+        ("string_value", "string_value"),
+        (123, "123"),
+        (True, "True"),
+    ],
+)
+def test_serialize_variation_value__return_expected(
+    value: object,
+    expected: str,
+) -> None:
+    # When
+    result = _serialize_variation_value(value)
+
+    # Then
+    assert result == expected