refactor: Update CI/QA environments to use serverless cache by default

Author: blarghmatey

src/ol_infrastructure/applications/edxapp/__main__.py

@@ -1,5 +1,4 @@
 # ruff: noqa: E501
-
 """Provision and deploy the resources needed for an edxapp installation.
 
 - Create S3 buckets required by edxapp
@@ -51,12 +50,12 @@
 from bridge.secrets.sops import read_yaml_secrets
 from bridge.settings.openedx.version_matrix import OpenLearningOpenEdxDeployment
 from ol_infrastructure.applications.edxapp.k8s_resources import create_k8s_resources
-from ol_infrastructure.components.aws.cache import OLAmazonCache, OLAmazonRedisConfig
 from ol_infrastructure.components.aws.database import OLAmazonDB, OLMariaDBConfig
 from ol_infrastructure.components.services.vault import (
     OLVaultDatabaseBackend,
     OLVaultMysqlDatabaseConfig,
 )
+from ol_infrastructure.lib.aws.cache_helper import create_redis_cache
 from ol_infrastructure.lib.aws.ec2_helper import (
     DiskTypes,
     InstanceTypes,
@@ -984,33 +983,26 @@ def cloud_init_user_data_func(
     vpc_id=edxapp_vpc_id,
 )
 
-redis_instance_type = (
-    redis_config.get("instance_type") or defaults(stack_info)["redis"]["instance_type"]
-)
-redis_cache_config = OLAmazonRedisConfig(
-    encrypt_transit=True,
-    auth_token=read_yaml_secrets(
-        Path(f"edxapp/{stack_info.env_prefix}.{stack_info.env_suffix}.yaml")
-    )["redis_auth_token"],
-    cluster_mode_enabled=False,
-    encrypted=True,
+# Create Redis cache (auto-selects serverless for CI/QA, dedicated for Prod)
+redis_defaults_config = defaults(stack_info)["redis"]
+redis_auth_token = read_yaml_secrets(
+    Path(f"edxapp/{stack_info.env_prefix}.{stack_info.env_suffix}.yaml")
+)["redis_auth_token"]
+
+edxapp_redis_cache = create_redis_cache(
+    stack_info=stack_info,
+    cache_name=f"edxapp-redis-{env_name}",
+    description="Redis cluster for edX platform tasks and caching",
+    security_group_ids=[redis_cluster_security_group.id],
+    subnet_group=edxapp_vpc["elasticache_subnet"],
+    subnet_ids=edxapp_vpc["subnet_ids"][:3],
+    auth_token=redis_auth_token,
     engine="valkey",
     engine_version="7.2",
-    instance_type=redis_instance_type,
+    instance_type=redis_config.get("instance_type")
+    or redis_defaults_config["instance_type"],
     num_instances=3,
-    shard_count=1,
-    auto_upgrade=True,
-    cluster_description="Redis cluster for edX platform tasks and caching",
-    cluster_name=f"edxapp-redis-{env_name}",
-    parameter_overrides={"maxmemory-policy": "allkeys-lru"},
-    security_groups=[redis_cluster_security_group.id],
-    subnet_group=edxapp_vpc[
-        "elasticache_subnet"
-    ],  # the name of the subnet group created in the OLVPC component resource
     tags=aws_config.tags,
-)
-edxapp_redis_cache = OLAmazonCache(
-    redis_cache_config,
     opts=ResourceOptions(
         aliases=[Alias(name=f"edxapp-redis-{env_name}-redis-elasticache-cluster")]
     ),
@@ -1026,7 +1018,7 @@ def cloud_init_user_data_func(
     "edxapp-redis-consul-service",
     node=edxapp_redis_consul_node.name,
     name="edxapp-redis",
-    port=redis_cache_config.port,
+    port=DEFAULT_REDIS_PORT,
     meta={
         "external-node": True,
         "external-probe": True,
@@ -1622,7 +1614,6 @@ def cloud_init_user_data_func(
     "fastly_access_logging_iam_role"
 )
 
-
 mfe_regex = "^/({})/".format("|".join(edxapp_mfe_paths))
 edxapp_fastly_service = fastly.ServiceVcl(
     f"fastly-{stack_info.env_prefix}-{stack_info.env_suffix}",

src/ol_infrastructure/applications/learn_ai/__main__.py

@@ -31,7 +31,6 @@
     ONE_MEGABYTE_BYTE,
 )
 from bridge.secrets.sops import read_yaml_secrets
-from ol_infrastructure.components.aws.cache import OLAmazonCache, OLAmazonRedisConfig
 from ol_infrastructure.components.aws.eks import OLEKSTrustRole, OLEKSTrustRoleConfig
 from ol_infrastructure.components.services import appdb
 from ol_infrastructure.components.services.k8s import (
@@ -53,6 +52,7 @@
     OLVaultK8SSecret,
     OLVaultK8SStaticSecretConfig,
 )
+from ol_infrastructure.lib.aws.cache_helper import create_redis_cache
 from ol_infrastructure.lib.aws.eks_helper import (
     check_cluster_namespace,
     default_psg_egress_args,
@@ -569,25 +569,22 @@
     tags=aws_config.tags,
 )
 
-redis_cache_config = OLAmazonRedisConfig(
-    encrypt_transit=True,
+# Create Redis cache (automatically selects serverless for CI, dedicated for Production)
+redis_defaults = defaults(stack_info)["redis"]
+redis_cache = create_redis_cache(
+    stack_info=stack_info,
+    cache_name=f"learn-ai-redis-{stack_info.env_suffix}",
+    description="Redis cluster for learn UI tasks and caching",
+    security_group_ids=[redis_cluster_security_group.id],
+    subnet_group=apps_vpc["elasticache_subnet"],
+    subnet_ids=apps_vpc["subnet_ids"][:3],
     auth_token=redis_config.require("password"),
-    cluster_mode_enabled=False,
-    encrypted=True,
-    engine_version="7.2",
     engine="valkey",
+    engine_version="7.2",
+    instance_type=redis_config.get("instance_type")
+    or redis_defaults.get("instance_type"),
     num_instances=3,
-    shard_count=1,
-    auto_upgrade=True,
-    cluster_description="Redis cluster for learn UI tasks and caching.",
-    cluster_name=f"learn-ai-redis-{stack_info.env_suffix}",
-    subnet_group=apps_vpc["elasticache_subnet"],
-    security_groups=[redis_cluster_security_group.id],
     tags=aws_config.tags,
-    **redis_defaults,
-)
-redis_cache = OLAmazonCache(
-    redis_cache_config,
     opts=ResourceOptions(
         aliases=[
             Alias(

src/ol_infrastructure/applications/mit_learn/__main__.py

@@ -29,10 +29,6 @@
 from ol_infrastructure.applications.mit_learn.k8s_secrets import (
     create_mitlearn_k8s_secrets,
 )
-from ol_infrastructure.components.aws.cache import (
-    OLAmazonCache,
-    OLAmazonRedisConfig,
-)
 from ol_infrastructure.components.aws.database import OLAmazonDB, OLPostgresDBConfig
 from ol_infrastructure.components.services.cert_manager import (
     OLCertManagerCert,
@@ -56,6 +52,7 @@
     OLVaultK8SResourcesConfig,
     OLVaultPostgresDatabaseConfig,
 )
+from ol_infrastructure.lib.aws.cache_helper import create_redis_cache
 from ol_infrastructure.lib.aws.eks_helper import (
     check_cluster_namespace,
     default_psg_egress_args,
@@ -1338,25 +1335,22 @@
     vpc_id=apps_vpc["id"],
     tags=aws_config.tags,
 )
-redis_cache_config = OLAmazonRedisConfig(
-    encrypt_transit=True,
+
+# Create Redis cache (auto-selects serverless for CI/QA, dedicated for Prod)
+redis_cache = create_redis_cache(
+    stack_info=stack_info,
+    cache_name=f"mitlearn-redis-{stack_info.env_suffix}",
+    description="Redis cluster for MIT Learn",
+    security_group_ids=[redis_cluster_security_group.id],
+    subnet_group=apps_vpc["elasticache_subnet"],
+    subnet_ids=apps_vpc["subnet_ids"][:3],
     auth_token=redis_config.require("password"),
-    cluster_mode_enabled=False,
-    encrypted=True,
-    engine_version="7.2",
     engine="valkey",
+    engine_version="7.2",
+    instance_type=redis_config.get("instance_type")
+    or redis_defaults.get("instance_type"),
     num_instances=3,
-    shard_count=1,
-    auto_upgrade=True,
-    cluster_description="Redis cluster for MIT Learn",
-    cluster_name=f"mitlearn-redis-{stack_info.env_suffix}",
-    subnet_group=apps_vpc["elasticache_subnet"],
-    security_groups=[redis_cluster_security_group.id],
     tags=aws_config.tags,
-    **redis_defaults,
-)
-redis_cache = OLAmazonCache(
-    redis_cache_config,
     opts=ResourceOptions(
         aliases=[
             Alias(

src/ol_infrastructure/applications/mit_learn/k8s_secrets.py

@@ -14,6 +14,7 @@
 
 from bridge.lib.magic_numbers import DEFAULT_REDIS_PORT
 from ol_infrastructure.components.aws.cache import OLAmazonCache
+from ol_infrastructure.components.aws.serverless_cache import OLAmazonServerlessCache
 from ol_infrastructure.components.services.vault import (
     OLVaultDatabaseBackend,
     OLVaultK8SDynamicSecretConfig,
@@ -133,7 +134,7 @@ def create_mitlearn_k8s_secrets(
     mitlearn_vault_mount: Mount,
     db_config: OLVaultDatabaseBackend,
     redis_password: str,
-    redis_cache: OLAmazonCache,
+    redis_cache: OLAmazonCache | OLAmazonServerlessCache,
 ) -> tuple[list[str], list[OLVaultK8SSecret | kubernetes.core.v1.Secret]]:
     """
     Create all Kubernetes secrets required by the mitlearn application.

src/ol_infrastructure/applications/mitxonline/__main__.py

@@ -28,15 +28,7 @@
 from ol_infrastructure.applications.mitxonline.k8s_secrets import (
     create_mitxonline_k8s_secrets,
 )
-from ol_infrastructure.components.aws.cache import (
-    OLAmazonCache,
-    OLAmazonRedisConfig,
-)
 from ol_infrastructure.components.aws.database import OLAmazonDB, OLPostgresDBConfig
-from ol_infrastructure.components.aws.serverless_cache import (
-    OLAmazonServerlessCache,
-    OLAmazonServerlessCacheConfig,
-)
 from ol_infrastructure.components.services.cert_manager import (
     OLCertManagerCert,
     OLCertManagerCertConfig,
@@ -59,6 +51,7 @@
     OLVaultK8SResourcesConfig,
     OLVaultPostgresDatabaseConfig,
 )
+from ol_infrastructure.lib.aws.cache_helper import create_redis_cache
 from ol_infrastructure.lib.aws.eks_helper import (
     check_cluster_namespace,
     default_psg_egress_args,
@@ -424,29 +417,22 @@
     vpc_id=apps_vpc["id"],
     tags=aws_config.tags,
 )
+# Create Redis cache (automatically selects serverless for CI/QA, dedicated for Production)
+# Configuration can override via redis:use_serverless_cache boolean
 redis_defaults = defaults(stack_info)["redis"]
-redis_defaults["instance_type"] = (
-    redis_config.get("instance_type") or redis_defaults["instance_type"]
-)
-redis_cache_config = OLAmazonRedisConfig(
-    encrypt_transit=True,
+redis_cache = create_redis_cache(
+    stack_info=stack_info,
+    cache_name=f"mitxonline-app-redis-{stack_info.env_suffix}",
+    description="Redis cluster for MITxonline",
+    security_group_ids=[redis_cluster_security_group.id],
+    subnet_group=apps_vpc["elasticache_subnet"],
+    subnet_ids=apps_vpc["subnet_ids"][:3],
     auth_token=redis_config.require("password"),
-    cluster_mode_enabled=False,
-    encrypted=True,
-    engine_version="7.2",
     engine="valkey",
+    engine_version="7.2",
+    instance_type=redis_config.get("instance_type") or redis_defaults["instance_type"],
     num_instances=3,
-    shard_count=1,
-    auto_upgrade=True,
-    cluster_description="Redis cluster for MITxonline",
-    cluster_name=f"mitxonline-app-redis-{stack_info.env_suffix}",
-    subnet_group=apps_vpc["elasticache_subnet"],
-    security_groups=[redis_cluster_security_group.id],
     tags=aws_config.tags,
-    **redis_defaults,
-)
-redis_cache = OLAmazonCache(
-    redis_cache_config,
     opts=ResourceOptions(
         aliases=[
             Alias(
@@ -456,24 +442,6 @@
     ),
 )
 
-serverless_cache_config = OLAmazonServerlessCacheConfig(
-    description="Redis cluster for MITxonline",
-    cache_name=f"mitxonline-app-serverless-{stack_info.env_suffix}",
-    security_group_ids=[redis_cluster_security_group.id],
-    subnet_ids=apps_vpc["subnet_ids"][:3],  # We can only specify three
-    tags=aws_config.tags,
-)
-serverless_cache = OLAmazonServerlessCache(
-    serverless_cache_config,
-    opts=ResourceOptions(
-        aliases=[
-            Alias(
-                name=f"mitxonline-app-serverless-{stack_info.env_suffix}-serverless-elasticache-cluster"
-            )
-        ]
-    ),
-)
-
 # Create Kubernetes secrets using the dedicated function
 # The function returns the names of the secrets and the Pulumi resource objects
 secret_names, secret_resources = create_mitxonline_k8s_secrets(

src/ol_infrastructure/applications/mitxonline/k8s_secrets.py

@@ -13,6 +13,7 @@
 
 from bridge.lib.magic_numbers import DEFAULT_REDIS_PORT
 from ol_infrastructure.components.aws.cache import OLAmazonCache
+from ol_infrastructure.components.aws.serverless_cache import OLAmazonServerlessCache
 from ol_infrastructure.components.services.vault import (
     OLVaultDatabaseBackend,
     OLVaultK8SDynamicSecretConfig,
@@ -133,7 +134,7 @@ def create_mitxonline_k8s_secrets(
     rds_endpoint: str,
     openedx_environment: str,
     redis_password: str,
-    redis_cache: OLAmazonCache,
+    redis_cache: OLAmazonCache | OLAmazonServerlessCache,
 ) -> tuple[list[str], list[OLVaultK8SSecret | kubernetes.core.v1.Secret]]:
     """
     Create all Kubernetes secrets required by the MITx Online application.

src/ol_infrastructure/applications/odl_video_service/__main__.py

@@ -30,7 +30,6 @@
     OLTargetGroupConfig,
     TagSpecification,
 )
-from ol_infrastructure.components.aws.cache import OLAmazonCache, OLAmazonRedisConfig
 from ol_infrastructure.components.aws.database import OLAmazonDB, OLPostgresDBConfig
 from ol_infrastructure.components.aws.mediaconvert import (
     MediaConvertConfig,
@@ -40,6 +39,7 @@
     OLVaultDatabaseBackend,
     OLVaultPostgresDatabaseConfig,
 )
+from ol_infrastructure.lib.aws.cache_helper import create_redis_cache
 from ol_infrastructure.lib.aws.ec2_helper import InstanceTypes, default_egress_args
 from ol_infrastructure.lib.aws.iam_helper import IAM_POLICY_VERSION, lint_iam_policy
 from ol_infrastructure.lib.consul import consul_key_helper, get_consul_provider
@@ -454,24 +454,22 @@
 redis_auth_token = secrets["redis"]["auth_token"]
 redis_config = Config("redis")
 
-ovs_server_redis_config = OLAmazonRedisConfig(
-    encrypt_transit=True,
+# Create Redis cache (automatically selects serverless for CI, dedicated for Production)
+redis_defaults = defaults(stack_info)["redis"]
+ovs_server_redis_cluster = create_redis_cache(
+    stack_info=stack_info,
+    cache_name=f"odl-video-service-redis-{stack_info.env_suffix}",
+    description="Redis cluster for ODL Video Service",
+    security_group_ids=[ovs_redis_security_group.id],
+    subnet_group=target_vpc["elasticache_subnet"],
+    subnet_ids=target_vpc["subnet_ids"][:3],
     auth_token=redis_auth_token,
-    engine_version="7.2",
     engine="valkey",
+    engine_version="7.2",
+    instance_type=redis_config.get("instance_type")
+    or redis_defaults.get("instance_type"),
     num_instances=3,
-    shard_count=1,
-    auto_upgrade=True,
-    cluster_mode_enabled=False,
-    cluster_description="Redis cluster for ODL Video Service.",
-    cluster_name=f"odl-video-service-redis-{stack_info.env_suffix}",
-    security_groups=[ovs_redis_security_group.id],
-    subnet_group=target_vpc["elasticache_subnet"],
     tags=aws_config.tags,
-    **defaults(stack_info)["redis"],
-)
-ovs_server_redis_cluster = OLAmazonCache(
-    ovs_server_redis_config,
     opts=ResourceOptions(
         aliases=[
             Alias(