Update plugin load_db to support air gapped environments

Signed-off-by: Dom Del Nano <ddelnano@gmail.com>

Author: ddelnano

k8s/cloud/public/base/plugin_db_updater_job.yaml

@@ -40,6 +40,8 @@ spec:
         volumeMounts:
         - name: certs
           mountPath: /certs
+        - name: plugin-configs
+          mountPath: /plugins
         env:
         - name: PL_POSTGRES_USERNAME
           valueFrom:
@@ -63,6 +65,8 @@ spec:
               key: PL_PLUGIN_SERVICE
         - name: PL_PLUGIN_REPO
           value: "pixie-io/pixie-plugin"
+        - name: PL_PLUGIN_DIR
+          value: "/plugins"
       restartPolicy: Never
       volumes:
       - name: pl-db-secrets
@@ -71,6 +75,15 @@ spec:
       - name: certs
         secret:
           secretName: service-tls-certs
+      # For airgapped installs, create this ConfigMap with plugin YAML files:
+      #   kubectl create configmap pl-plugin-configs --from-file=plugins/
+      # Where plugins/ has subdirectories per plugin containing plugin.yaml
+      # and retention.yaml. If the ConfigMap doesn't exist, the job falls
+      # back to fetching from GitHub.
+      - name: plugin-configs
+        configMap:
+          name: pl-plugin-configs
+          optional: true
   backoffLimit: 1
   parallelism: 1
   completions: 1

src/cloud/indexer/BUILD.bazel

@@ -44,6 +44,7 @@ go_library(
     ],
 )
 
+# Dummy chan
 pl_go_binary(
     name = "indexer_server",
     embed = [":indexer_lib"],

src/cloud/indexer/indexer_server.go

@@ -51,7 +51,7 @@ func init() {
 
 	pflag.String("md_index_name", "", "The elastic index name for metadata.")
 	pflag.String("md_index_max_age", "", "The amount of time before rolling over the elastic index as a string, eg '30d'")
-	pflag.String("md_index_delete_after", "", "The amount of time after rollover to delete old elastic indices, as a string, eg '30d'")
+	pflag.String("md_index_delete_after", "", "The amount of time after rollover to delete old elastic indices, as a string, eg '30d'.")
 	pflag.Int("md_index_replicas", 4, "The number of replicas to setup for the metadata index.")
 	pflag.Bool("md_manual_index_management", false, "Skip creation of managed elastic indices. Requires manually deploying an elastic index with md_index_name")
 }

src/cloud/plugin/load_db/main.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"path/filepath"
 
 	"github.com/blang/semver"
@@ -56,6 +57,7 @@ const githubArchiveTmpl = "https://api.github.com/repos/%s/tarball"
 // bazel run //src/cloud/plugin/load_db:push_plugin_db_updater_image
 func init() {
 	pflag.String("plugin_repo", "pixie-io/pixie-plugin", "The name of the plugin repo.")
+	pflag.String("plugin_dir", "", "Local directory containing plugin configs. When set, plugins are loaded from this directory instead of fetching from GitHub. Expected structure: <plugin_dir>/<plugin_name>/{plugin.yaml,retention.yaml}")
 	pflag.String("plugin_service", "plugin-service.plc.svc.cluster.local:50600", "The plugin service url (load balancer/list is ok)")
 	pflag.String("domain_name", "dev.withpixie.dev", "The domain name of Pixie Cloud")
 }
@@ -92,11 +94,55 @@ type configSet struct {
 var configsByPath = make(map[string]*configSet)
 
 func loadPlugins(db *sqlx.DB) {
+	pluginDir := viper.GetString("plugin_dir")
+	if pluginDir != "" {
+		if loadPluginsFromDir(pluginDir, db) {
+			log.Infof("Loaded plugins from local directory: %s", pluginDir)
+			return
+		}
+		log.Infof("Plugin directory %s is empty or missing, falling back to GitHub", pluginDir)
+	}
+
 	pluginRepo := viper.GetString("plugin_repo")
 	if pluginRepo == "" {
-		log.Fatal("Must specify --plugin_repo")
+		log.Fatal("Must specify --plugin_repo or --plugin_dir")
 	}
 
+	loadPluginsFromGitHub(pluginRepo, db)
+}
+
+// loadPluginsFromDir walks a local directory for plugin YAML files.
+// Returns true if any files were found and processed.
+func loadPluginsFromDir(dir string, db *sqlx.DB) bool {
+	info, err := os.Stat(dir)
+	if err != nil || !info.IsDir() {
+		return false
+	}
+
+	found := false
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		f, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("failed to open %s: %w", path, err)
+		}
+		defer f.Close()
+		found = true
+		processFile(path, f, db)
+		return nil
+	})
+	if err != nil {
+		log.WithError(err).Fatal("Failed to read plugin directory")
+	}
+	return found
+}
+
+func loadPluginsFromGitHub(pluginRepo string, db *sqlx.DB) {
 	client := http.Client{}
 	req, err := http.NewRequest("GET", fmt.Sprintf(githubArchiveTmpl, pluginRepo), nil)
 	if err != nil {