Replace Deployment update steps joins with lazy subquery

Author: korbajan

rest-service/manager_rest/storage/resource_models.py

@@ -2025,7 +2025,7 @@ def to_response(self, include=None, **kwargs):
         dep_update_dict = super(DeploymentUpdate, self).to_response(
             include, **kwargs)
         if 'steps' in include:
-            dep_update_dict['steps'] = [step.to_dict() for step in self.steps]
+            dep_update_dict['steps'] = self.steps
         if 'recursive_dependencies' in include:
             dep_update_dict['recursive_dependencies'] = \
                 self.recursive_dependencies
@@ -2057,8 +2057,10 @@ class DeploymentUpdateStep(SQLResourceBase):
     @declared_attr
     def deployment_update(cls):
         return one_to_many_relationship(
-            cls, DeploymentUpdate, cls._deployment_update_fk,
-            backref=db.backref('steps', cascade='all'))
+            cls,
+            DeploymentUpdate,
+            cls._deployment_update_fk,
+        )
 
     deployment_update_id = association_proxy('deployment_update', 'id')
 
@@ -2091,6 +2093,43 @@ def __lt__(self, other):
         return False
 
 
+# Define the steps as a property of the DeploymentUpdate
+# It is required to define it here after DeploymentUpdateStep
+# is loaded by SQLAlchemy.
+# In big deployments with lots of steps standard JOIN causes
+# OOMKilled error on pod running by Kubernetes.
+# This declaration adds a custom property that executes lazy select query.
+# The goal of this is to mitigate memory consumption issue.
+DeploymentUpdate.steps = db.column_property(
+    db.cast(
+        db.func.array(
+
+            db.select([
+                db.func.jsonb_build_object(
+                    'id', DeploymentUpdateStep.id,
+                    'action', DeploymentUpdateStep.action,
+                    'entity_id', DeploymentUpdateStep.entity_id,
+                    'entity_type', DeploymentUpdateStep.entity_type,
+
+                    'topology_order', DeploymentUpdateStep.topology_order,
+                    'private_resource', DeploymentUpdateStep.private_resource,
+
+                    'visibility', DeploymentUpdateStep.visibility,
+
+                )
+            ])
+            .where(
+                DeploymentUpdateStep._deployment_update_fk ==
+                DeploymentUpdate._storage_id
+            )
+
+        ),
+        db.ARRAY(db.JSON),
+    ).label('steps')
+
+)
+
+
 class DeploymentModification(CreatedAtMixin, SQLResourceBase):
     __tablename__ = 'deployment_modifications'
 

tests/integration_tests/tests/agentless_tests/test_deployment_workflows.py

@@ -128,3 +128,23 @@ def test_delete_botched_deployment(self):
             self.fail("Expected deployment to be deleted")
         except CloudifyClientError as e:
             self.assertEqual(404, e.status_code)
+
+    def test_deployment_workflows_update_steps(self) -> None:
+        dsl_path = resource("dsl/custom_workflow_mapping.yaml")
+        deployment, _ = self.deploy_application(dsl_path)
+        deployment_id = deployment.id
+
+        deployment = self.client.deployments.get(deployment_id)
+        deployment_updates = self.client.deployment_updates.list(
+            deployment_id=deployment_id, _include=['id', 'steps']
+        )
+
+        for step in deployment_updates.items[0].steps:
+            assert 'id' in step
+
+            assert 'action' in step and step['action'] is not None
+            assert 'entity_id' in step and step['entity_id'] is not None
+            assert 'entity_type' in step and step['entity_type'] is not None
+            assert 'topology_order' in step and step['topology_order'] is not None
+            assert 'private_resource' in step and step['private_resource'] is not None
+            assert 'visibility' in step and step['visibility'] is not None