Implement GET /v3/processes/:guid/process_instances and embed=process_instances (#4796)

* Implement GET /v3/processes/:guid/process_instances

This endpoint returns information for all process instances (= actual
LRPs) for the given process. Per process instance the following
information is shown: index, state, since.

The data is retrieved from Diego via the endpoint
'/v1/actual_lrps/list_by_process_guids'.

The InstancesStatsReporter processes this data as follows:
- group by process guid
- sort / group by index
- keep only the newest LRP per index
- fill missing instances (i.e. less actual LRPs than desired) with DOWN
  state

Additional changes:
- BbsInstancesClient: handle_diego_errors needs check for
  ResourceNotFound only when calling desired_lrp_by_process_guid. The
  other methods return an empty list.
- BuildpackPresenter, ProcessPresenter: remove useless method

Co-authored-by: Sven Krieger <sven.krieger@sap.com>

* Implement embed=process_instances

Process instances can be embedded into process resources; the parameter
embed=process_instances is available for the following API endpoints:
- GET /v3/processes/:guid?embed=process_instances
- GET /v3/apps/:guid/processes/:type?embed=process_instances
- GET /v3/processes?embed=process_instances
- GET /v3/apps/:guid/processes?embed=process_instances

A new ProcessShowMessage has been introduced; ProcessesListMessage has
been extended. Both messages allow the 'embed' key that is checked with
the EmbedParamValidator.

The process output is enhanced by the EmbedProcessInstancesDecorator
that fetches the required information from Diego and adds a field
'process_instances' to the resource.

Process resources now also contain a link to 'process_instances'.

API docs have been extended with the new process instances
pseudo-resource and a description of the 'embed' parameter.

The 'embed' parameter is also described in the V3 style guide.

---------

Co-authored-by: Sven Krieger <sven.krieger@sap.com>

Author: philippthun

.rubocop_todo.yml

@@ -85,8 +85,6 @@ Lint/UnusedMethodArgument:
 Lint/UselessMethodDefinition:
   Exclude:
     - 'app/messages/route_destination_update_message.rb'
-    - 'app/presenters/v3/buildpack_presenter.rb'
-    - 'app/presenters/v3/process_presenter.rb'
     - 'spec/support/fake_front_controller.rb'
 
 # Offense count: 791

app/controllers/v3/processes_controller.rb

@@ -1,6 +1,8 @@
 require 'presenters/v3/paginated_list_presenter'
 require 'presenters/v3/process_presenter'
 require 'presenters/v3/process_stats_presenter'
+require 'presenters/v3/process_instances_presenter'
+require 'decorators/embed_process_instances_decorator'
 require 'cloud_controller/paging/pagination_options'
 require 'actions/process_delete'
 require 'fetchers/process_list_fetcher'
@@ -9,6 +11,7 @@
 require 'actions/process_terminate'
 require 'actions/process_update'
 require 'messages/process_scale_message'
+require 'messages/process_show_message'
 require 'messages/process_update_message'
 require 'messages/processes_list_message'
 require 'controllers/v3/mixins/app_sub_resource'
@@ -40,17 +43,30 @@ def index
                 end
     end
 
+    decorators = []
+    decorators << EmbedProcessInstancesDecorator if EmbedProcessInstancesDecorator.match?(message.embed)
+
     render status: :ok, json: Presenters::V3::PaginatedListPresenter.new(
       presenter: Presenters::V3::ProcessPresenter,
       paginated_result: SequelPaginator.new.get_page(dataset, message.try(:pagination_options)),
       path: base_url(resource: 'processes'),
-      message: message
+      message: message,
+      decorators: decorators
     )
   end
 
   def show
-    # TODO
-    render status: :ok, json: Presenters::V3::ProcessPresenter.new(@process, show_secrets: permission_queryer.can_read_secrets_in_space?(@space.id, @space.organization_id))
+    message = ProcessShowMessage.from_params(query_params)
+    invalid_param!(message.errors.full_messages) unless message.valid?
+
+    decorators = []
+    decorators << EmbedProcessInstancesDecorator if EmbedProcessInstancesDecorator.match?(message.embed)
+
+    render status: :ok, json: Presenters::V3::ProcessPresenter.new(
+      @process,
+      show_secrets: permission_queryer.can_read_secrets_in_space?(@space.id, @space.organization_id),
+      decorators: decorators
+    )
   end
 
   def update
@@ -106,6 +122,12 @@ def stats
     render status: :ok, json: Presenters::V3::ProcessStatsPresenter.new(@process.type, process_stats)
   end
 
+  def process_instances
+    instances = instances_reporters.instances_for_processes([@process])
+
+    render status: :ok, json: Presenters::V3::ProcessInstancesPresenter.new(instances[@process.guid], @process)
+  end
+
   private
 
   def find_process_and_space

app/decorators/embed_process_instances_decorator.rb

@@ -0,0 +1,48 @@
+module VCAP::CloudController
+  class EmbedProcessInstancesDecorator
+    class << self
+      def match?(embed)
+        embed&.include?('process_instances')
+      end
+
+      def decorate(hash, processes)
+        instances_reporters = CloudController::DependencyLocator.instance.instances_reporters
+        instances = instances_reporters.instances_for_processes(processes)
+
+        if hash.key?(:resources)
+          # Decorate PaginatedListPresenter
+          processes.each do |process|
+            resource_index = hash[:resources].find_index { |resource| resource[:guid] == process.guid }
+            next unless resource_index # Should not happen...
+
+            hash[:resources][resource_index] = embed_process_instances(hash[:resources][resource_index], process_instances(instances, process.guid))
+          end
+        else
+          # Decorate ProcessPresenter
+          hash = embed_process_instances(hash, process_instances(instances, hash[:guid]))
+        end
+
+        hash
+      end
+
+      private
+
+      def process_instances(instances, process_guid)
+        instances[process_guid]&.map do |index, instance|
+          {
+            index: index,
+            state: instance[:state],
+            since: instance[:since]
+          }
+        end || []
+      end
+
+      def embed_process_instances(resource_hash, process_instances)
+        hash_as_array = resource_hash.to_a
+        before_relationships = hash_as_array.index { |k, _| k == :relationships } || hash_as_array.length
+        hash_as_array.insert(before_relationships, [:process_instances, process_instances])
+        hash_as_array.to_h
+      end
+    end
+  end
+end

app/messages/base_message.rb

@@ -139,6 +139,22 @@ def validate(record)
       end
     end
 
+    class EmbedParamValidator < ActiveModel::Validator
+      def validate(record)
+        return unless record.requested?(:embed)
+
+        key_counts = Hash.new(0)
+        record.embed.each do |embed_candidate|
+          if options[:valid_values].member?(embed_candidate)
+            key_counts[embed_candidate] += 1
+            record.errors.add(:base, message: "Duplicate embedded resource: '#{embed_candidate}'") if key_counts[embed_candidate] == 2
+          else
+            record.errors.add(:base, message: "Invalid embedded resource: '#{embed_candidate}'. Valid embedded resources are: '#{options[:valid_values].join("', '")}'")
+          end
+        end
+      end
+    end
+
     class LifecycleTypeParamValidator < ActiveModel::Validator
       def validate(record)
         return unless record.requested?(:lifecycle_type)

app/messages/process_show_message.rb

@@ -0,0 +1,14 @@
+require 'messages/base_message'
+
+module VCAP::CloudController
+  class ProcessShowMessage < BaseMessage
+    register_allowed_keys [:embed]
+
+    validates_with NoAdditionalParamsValidator
+    validates_with EmbedParamValidator, valid_values: ['process_instances']
+
+    def self.from_params(params)
+      super(params, %w[embed])
+    end
+  end
+end

app/messages/processes_list_message.rb

@@ -8,15 +8,19 @@ class ProcessesListMessage < MetadataListMessage
       space_guids
       organization_guids
       app_guids
+      embed
     ]
 
     validates_with NoAdditionalParamsValidator # from BaseMessage
+    validates_with EmbedParamValidator, valid_values: ['process_instances']
 
+    validates :space_guids, array: true, allow_nil: true
+    validates :organization_guids, array: true, allow_nil: true
     validates :app_guids, array: true, allow_nil: true
     validate :app_nested_request, if: -> { app_guid.present? }
 
     def self.from_params(params)
-      super(params, %w[types space_guids organization_guids app_guids])
+      super(params, %w[types space_guids organization_guids app_guids embed])
     end
 
     def to_param_hash

app/presenters/v3/buildpack_presenter.rb

@@ -26,13 +26,6 @@ def to_hash
       }
     end
 
-    class << self
-      # :labels and :annotations come from MetadataPresentationHelpers
-      def associated_resources
-        super
-      end
-    end
-
     private
 
     def buildpack

app/presenters/v3/process_instances_presenter.rb

@@ -0,0 +1,47 @@
+require 'presenters/v3/base_presenter'
+require 'presenters/mixins/metadata_presentation_helpers'
+
+module VCAP::CloudController
+  module Presenters
+    module V3
+      class ProcessInstancesPresenter < BasePresenter
+        attr_reader :process
+
+        def initialize(instances, process)
+          super(instances)
+          @process = process
+        end
+
+        def to_hash
+          {
+            resources: build_instances,
+            links: build_links
+          }
+        end
+
+        private
+
+        def instances
+          @resource
+        end
+
+        def build_instances
+          instances.map do |index, instance|
+            {
+              index: index,
+              state: instance[:state],
+              since: instance[:since]
+            }
+          end
+        end
+
+        def build_links
+          {
+            self: { href: url_builder.build_url(path: "/v3/processes/#{process.guid}/process_instances") },
+            process: { href: url_builder.build_url(path: "/v3/processes/#{process.guid}") }
+          }
+        end
+      end
+    end
+  end
+end

app/presenters/v3/process_presenter.rb

@@ -8,20 +8,14 @@ module V3
       class ProcessPresenter < BasePresenter
         include VCAP::CloudController::Presenters::Mixins::MetadataPresentationHelpers
 
-        class << self
-          # :labels and :annotations come from MetadataPresentationHelpers
-          def associated_resources
-            super
-          end
-        end
-
         def to_hash
           health_check_data = { timeout: process.health_check_timeout, invocation_timeout: process.health_check_invocation_timeout, interval: process.health_check_interval }
           health_check_data[:endpoint] = process.health_check_http_endpoint if process.health_check_type == HealthCheckTypes::HTTP
 
           readiness_health_check_data = { invocation_timeout: process.readiness_health_check_invocation_timeout, interval: process.readiness_health_check_interval }
           readiness_health_check_data[:endpoint] = process.readiness_health_check_http_endpoint if process.readiness_health_check_type == HealthCheckTypes::HTTP
-          {
+
+          hash = {
             guid: process.guid,
             created_at: process.created_at,
             updated_at: process.updated_at,
@@ -51,6 +45,8 @@ def to_hash
             },
             links: build_links
           }
+
+          @decorators.reduce(hash) { |memo, d| d.decorate(memo, [process]) }
         end
 
         private
@@ -73,7 +69,8 @@ def build_links
             scale: { href: url_builder.build_url(path: "/v3/processes/#{process.guid}/actions/scale"), method: 'POST' },
             app: { href: url_builder.build_url(path: "/v3/apps/#{process.app_guid}") },
             space: { href: url_builder.build_url(path: "/v3/spaces/#{process.space_guid}") },
-            stats: { href: url_builder.build_url(path: "/v3/processes/#{process.guid}/stats") }
+            stats: { href: url_builder.build_url(path: "/v3/processes/#{process.guid}/stats") },
+            process_instances: { href: url_builder.build_url(path: "/v3/processes/#{process.guid}/process_instances") }
           }
         end
       end

config/routes.rb

@@ -55,6 +55,7 @@
   get '/processes', to: 'processes#index'
   get '/processes/:process_guid', to: 'processes#show'
   patch '/processes/:process_guid', to: 'processes#update'
+  get '/processes/:process_guid/process_instances', to: 'processes#process_instances'
   delete '/processes/:process_guid/instances/:index', to: 'processes#terminate'
   post '/processes/:process_guid/actions/scale', to: 'processes#scale'
   get '/processes/:process_guid/stats', to: 'processes#stats'