Merge pull request #16 from code0-tech/add-support-for-reticulum-builds-with-version-overrides

Add command to trigger reticulum build with version overrides

Author: Taucher2003

lib/pyxis/commands/components.rb

@@ -19,6 +19,28 @@ def info
         result
       end
 
+      desc 'build', 'Start a reticulum build with specific versions'
+      method_option :aquila_sha, desc: 'Commit SHA of aquila to build', type: :string
+      method_option :draco_sha, desc: 'Commit SHA of draco to build', type: :string
+      method_option :sagittarius_sha, desc: 'Commit SHA of sagittarius to build', type: :string
+      method_option :sculptor_sha, desc: 'Commit SHA of sculptor to build', type: :string
+      method_option :taurus_sha, desc: 'Commit SHA of taurus to build', type: :string
+      def build
+        version_overrides = {
+          aquila: options[:aquila_sha],
+          draco: options[:draco_sha],
+          sagittarius: options[:sagittarius_sha],
+          sculptor: options[:sculptor_sha],
+          taurus: options[:taurus_sha],
+        }.compact
+
+        pipeline = Pyxis::Services::CreateReticulumBuildService.new(version_overrides).execute
+
+        raise Pyxis::MessageError, 'Failed to create pipeline' if pipeline.nil?
+
+        "Created reticulum build at #{pipeline.web_url}"
+      end
+
       desc 'update', 'Update a component in reticulum'
       method_option :component, aliases: '-c', desc: 'The component to update', required: true, type: :string
       def update

lib/pyxis/dry_run_enforcer.rb

@@ -3,7 +3,7 @@
 module Pyxis
   module DryRunEnforcer
     class FaradayBlocker < Faraday::Middleware
-      DryRunError = Class.new(StandardError)
+      DryRunError = Class.new(Pyxis::Error)
       include ::SemanticLogger::Loggable
 
       def on_request(env)

lib/pyxis/generic_faraday.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Pyxis
+  class GenericFaraday
+    include SemanticLogger::Loggable
+
+    def self.create(options)
+      faraday = Faraday.new(options)
+      faraday.use Pyxis::DryRunEnforcer::FaradayBlocker
+      faraday.use Pyxis::Logger::FaradayLogger
+
+      enhance_faraday(faraday)
+
+      faraday
+    end
+
+    def self.enhance_faraday(faraday)
+      %i[get post put patch delete].each do |method|
+        faraday.define_singleton_method(:"#{method}_json") do |*args, **kwargs|
+          response = faraday.send(method, *args, **kwargs)
+          json = response.body.blank? ? nil : JSON.parse(response.body)
+
+          Thor::CoreExt::HashWithIndifferentAccess.new(
+            {
+              body: json.is_a?(Hash) ? Thor::CoreExt::HashWithIndifferentAccess.new(json) : json,
+              response: response,
+            }
+          )
+        end
+      end
+    end
+  end
+end

lib/pyxis/github_client.rb

@@ -31,18 +31,26 @@ def without_auto_pagination(octokit)
         octokit.instance_variable_set(:@auto_paginate, current_auto_paginate)
       end
 
-      private
-
-      def create_octokit(instance)
-        logger.info('Creating octokit client', instance: instance)
+      def create_installation_access_token(instance, options = {})
         config = CLIENT_CONFIGS[instance]
         global_client = Octokit::Client.new(bearer_token: create_jwt(config[:private_key_location], config[:app_id]))
         logger.debug('Created JWT for client', app: global_client.app.slug)
         installation_token = Pyxis::GlobalStatus.with_faraday_dry_run_bypass do
-          global_client.create_app_installation_access_token(config[:installation_id])
+          global_client.create_app_installation_access_token(config[:installation_id], options)
         end
+
         logger.debug('Created app installation access token',
                      installation_token: installation_token.to_h.except(:token))
+
+        installation_token
+      end
+
+      private
+
+      def create_octokit(instance)
+        logger.info('Creating octokit client', instance: instance)
+
+        installation_token = create_installation_access_token(instance)
         Octokit::Client.new(bearer_token: installation_token[:token])
       end
 

lib/pyxis/gitlab_client.rb

@@ -27,30 +27,48 @@ def self.create_client(instance)
           'Private-Token': File.read(client_config[:private_token]),
         },
       }
-      faraday = Faraday.new(options)
-      faraday.use Pyxis::DryRunEnforcer::FaradayBlocker
-      faraday.use Pyxis::Logger::FaradayLogger
 
-      enhance_faraday(faraday)
+      GenericFaraday.create(options)
+    end
+
+    def self.paginate_json(client, url, options = {})
+      response = []
+
+      client.get_json(url, options).tap do |page|
+        response += page.body
+        while (next_page_link = PageLinks.new(page.response.env.response_headers).next)
+          page = client.get_json(next_page_link)
+          response += page.body
+        end
+      end
 
-      faraday
+      response
     end
 
-    def self.enhance_faraday(faraday)
-      %i[get post put patch delete].each do |method|
-        faraday.define_singleton_method(:"#{method}_json") do |*args, **kwargs|
-          response = faraday.send(method, *args, **kwargs)
-          json = response.body.blank? ? nil : JSON.parse(response.body)
-
-          if json.is_a?(Hash)
-            Thor::CoreExt::HashWithIndifferentAccess.new(
-              {
-                body: Thor::CoreExt::HashWithIndifferentAccess.new(json),
-                response: response,
-              }
-            )
-          else
-            response
+    class PageLinks
+      HEADER_LINK = 'link'
+      DELIM_LINKS = ','
+      LINK_REGEX = /<([^>]+)>; rel="([^"]+)"/
+      METAS = %w[last next first prev].freeze
+
+      attr_accessor(*METAS)
+
+      def initialize(headers)
+        link_header = headers[HEADER_LINK]
+
+        extract_links(link_header) if link_header && link_header =~ /(next|first|last|prev)/
+      end
+
+      private
+
+      def extract_links(header)
+        header.split(DELIM_LINKS).each do |link|
+          LINK_REGEX.match(link.strip) do |match|
+            url = match[1]
+            meta = match[2]
+            next if !url || !meta || METAS.index(meta).nil?
+
+            send("#{meta}=", url)
           end
         end
       end

lib/pyxis/managed_versioning/component_info.rb

@@ -17,25 +17,77 @@ def execute
         )
         return nil if pipeline.response.status == 404
 
-        reticulum_sha = pipeline.body.sha
-
-        components = {}
-
-        Pyxis::Project.components.each do |project_name|
-          component_project_class = Pyxis::Project.const_get(project_name)
-          version_file = "versions/#{component_project_class.component_name}"
-
-          begin
-            version_content = GithubClient.octokit.contents(Project::Reticulum.github_path, path: version_file,
-                                                                                            ref: reticulum_sha)
-            version = Base64.decode64(version_content.content)
-            components[component_project_class.component_name] = version
-          rescue Octokit::NotFound
-            logger.warn("Version file not found for #{component_project_class.component_name} at SHA #{reticulum_sha}")
-          end
+        jobs = GitlabClient.paginate_json(
+          GitlabClient.client,
+          "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{build_id}/jobs"
+        )
+
+        manifests = find_manifests(jobs)
+
+        components = {
+          reticulum: pipeline.body.sha,
+        }
+
+        manifests.each do |image|
+          component = image.first.to_sym
+          next if components.key?(component)
+
+          image_tag = image.length == 1 ? build_id : "#{build_id}-#{image.last}"
+
+          components[component] = revision_for("code0-tech/reticulum/ci-builds/#{component}", image_tag)
         end
 
-        components
+        components.compact
+      end
+
+      private
+
+      def ghcr_client
+        @ghcr_client ||= GenericFaraday.create({ url: 'https://ghcr.io' })
+      end
+
+      def token_for(image)
+        response = ghcr_client.get_json(
+          'token',
+          {
+            scope: "repository:#{image}:pull",
+            service: 'ghcr.io',
+          }
+        )
+
+        logger.warn('Failed to retrieve token', image: image) unless response.response.status == 200
+
+        response.body.token
+      end
+
+      def revision_for(image, tag)
+        token = token_for(image)
+
+        response = ghcr_client.get_json(
+          "v2/#{image}/manifests/#{tag}",
+          {},
+          {
+            Authorization: "Bearer #{token}",
+            Accept: 'application/vnd.oci.image.index.v1+json',
+          }
+        )
+
+        logger.warn('Failed to retrieve tag for image', image: image, tag: tag) unless response.response.status == 200
+
+        response.body.annotations&.[]('org.opencontainers.image.version')
+      end
+
+      def find_manifests(jobs)
+        jobs.map { |job| job['name'] }
+            .select { |job| job.start_with?('manifest:') }
+            .map { |job| job.delete_prefix('manifest:') }
+            .sort
+            .map { |job| job.split(': ') }
+            .map do |image|
+          next image if image.length == 1
+
+          [image.first, image.last.delete_prefix('[').delete_suffix(']')]
+        end
       end
     end
   end

lib/pyxis/project/base.rb

@@ -16,6 +16,10 @@ def github_path
           paths[:github]
         end
 
+        def github_repository_name
+          github_path.split('/')[1]
+        end
+
         def api_gitlab_path
           paths[:gitlab].gsub('/', '%2F')
         end

lib/pyxis/services/create_reticulum_build_service.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Pyxis
+  module Services
+    class CreateReticulumBuildService
+      InvalidVersionOverride = Class.new(Pyxis::Error)
+
+      include SemanticLogger::Loggable
+
+      attr_reader :version_overrides
+
+      def initialize(version_overrides)
+        @version_overrides = version_overrides
+      end
+
+      def execute
+        logger.info('Creating build with version overrides', version_overrides: version_overrides)
+
+        version_overrides.each_pair do |component, version|
+          validate_override!(component, version)
+        end
+
+        pipeline = GitlabClient.client.post_json(
+          "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipeline",
+          {
+            ref: Project::Reticulum.default_branch,
+            variables: version_override_variables + token_variable,
+          }
+        )
+
+        pipeline.body if pipeline.response.status == 201
+      end
+
+      private
+
+      def validate_override!(component, version)
+        project = Pyxis::Project.const_get(component.capitalize)
+
+        begin
+          GithubClient.octokit.tag(project.github_path, version)
+        rescue Octokit::UnprocessableEntity, Octokit::NotFound
+          begin
+            GithubClient.octokit.commit(project.github_path, version)
+          rescue Octokit::UnprocessableEntity, Octokit::NotFound
+            raise InvalidVersionOverride, "Invalid version '#{version}' for component '#{component}'"
+          end
+        end
+      end
+
+      def version_override_variables
+        variables = []
+
+        version_overrides.each_pair do |component, version|
+          variables << {
+            key: "OVERRIDE_#{component}_VERSION",
+            value: version,
+          }
+        end
+
+        variables
+      end
+
+      def token_variable
+        [
+          {
+            key: 'C0_GH_TOKEN',
+            value: File.read(ENV.fetch('PYXIS_GH_RETICULUM_PUBLISH_TOKEN')),
+          }
+        ]
+      end
+    end
+  end
+end