Merge pull request #42 from flanksource/issue-41

feat: find and update resource based on the object key

Author: moshloop

.github/workflows/test.yml

@@ -4,10 +4,10 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.13.x]
+        go-version: [1.14.x]
         platform: [ubuntu-latest]
         k8s:
-          - v1.16.9
+          - v1.20.7
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Install Go

api/v1/gitopsapi_types.go

@@ -57,8 +57,9 @@ type GitopsAPISpec struct {
 	Kustomization string `json:"kustomization,omitempty"`
 
 	// The path to save the resource into, should including templating to make it unique per cluster/namespace/kind/name tuple e.g. `specs/clusters/{{.cluster}}/{{.name}}.yaml`
-	// +required
 	Path string `json:"path,omitempty"`
+	// SearchPath defines the subdir in which the matching object needs to be searched. In case Path and SearchPath both are defined SearchPath takes precedence
+	SearchPath string `json:"searchPath,omitempty"`
 }
 
 type PullRequestTemplate struct {

config/crd/bases/git.flanksource.com_gitopsapis.yaml

@@ -86,6 +86,11 @@ spec:
                 items:
                   type: string
                 type: array
+              searchPath:
+                description: SearchPath defines the subdir in which the matching object
+                  needs to be searched. In case Path and SearchPath both are defined
+                  SearchPath takes precedence
+                type: string
               secretRef:
                 description: The secret name containing the Git credentials. For SSH
                   repositories the secret must contain SSH_PRIVATE_KEY, SSH_PRIVATE_KEY_PASSORD

config/deploy/crd.yml

@@ -337,6 +337,11 @@ spec:
                 items:
                   type: string
                 type: array
+              searchPath:
+                description: SearchPath defines the subdir in which the matching object
+                  needs to be searched. In case Path and SearchPath both are defined
+                  SearchPath takes precedence
+                type: string
               secretRef:
                 description: The secret name containing the Git credentials. For SSH
                   repositories the secret must contain SSH_PRIVATE_KEY, SSH_PRIVATE_KEY_PASSORD

config/deploy/operator.yml

@@ -337,6 +337,11 @@ spec:
                 items:
                   type: string
                 type: array
+              searchPath:
+                description: SearchPath defines the subdir in which the matching object
+                  needs to be searched. In case Path and SearchPath both are defined
+                  SearchPath takes precedence
+                type: string
               secretRef:
                 description: The secret name containing the Git credentials. For SSH
                   repositories the secret must contain SSH_PRIVATE_KEY, SSH_PRIVATE_KEY_PASSORD

controllers/gitopsapi_controller.go

@@ -23,7 +23,9 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -141,6 +143,8 @@ func HandleGitopsAPI(ctx context.Context, logger logr.Logger, git connectors.Con
 	if err = json.Unmarshal(body, &obj.Object); err != nil {
 		return hash, pr, errors.WithStack(err)
 	}
+	objKey := fmt.Sprintf("%s-%s-%s", obj.GetName(), obj.GetNamespace(), obj.GetKind())
+	contentPath := fmt.Sprintf("%s-%s-%s.yaml", obj.GetKind(), obj.GetNamespace(), obj.GetName())
 	api.Spec.Branch, err = text.Template(api.Spec.Branch, obj.Object)
 	if err != nil {
 		return
@@ -149,11 +153,47 @@ func HandleGitopsAPI(ctx context.Context, logger logr.Logger, git connectors.Con
 	if err != nil {
 		return
 	}
-
-	if api.Spec.Path == "" {
-		api.Spec.Path = fmt.Sprintf("%s-%s-%s.yaml", obj.GetKind(), obj.GetNamespace(), obj.GetName())
+	if api.Spec.SearchPath != "" {
+		repoRoot := fs.Root()
+		if strings.HasSuffix(repoRoot, "/") {
+			api.Spec.SearchPath = repoRoot + api.Spec.SearchPath
+		} else {
+			api.Spec.SearchPath = repoRoot + "/" + api.Spec.SearchPath
+		}
+		if err := filepath.Walk(api.Spec.SearchPath, func(filePath string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.Name() == "kustomization.yaml" || info.IsDir() {
+				return nil
+			}
+			if path.Ext(filePath) == ".yaml" || path.Ext(filePath) == ".yml" {
+				resource := unstructured.Unstructured{Object: make(map[string]interface{})}
+				buf, err := ioutil.ReadFile(filePath)
+				if err != nil {
+					return err
+				}
+				if err := yaml.Unmarshal(buf, &resource); err != nil {
+					return err
+				}
+				resourceKey := fmt.Sprintf("%s-%s-%s", resource.GetName(), resource.GetNamespace(), resource.GetKind())
+				if objKey == resourceKey {
+					contentPath, err = filepath.Rel(repoRoot, filePath)
+					if err != nil {
+						return err
+					}
+					return nil
+				}
+			}
+			return nil
+		}); err != nil {
+			return hash, pr, errors.WithStack(err)
+		}
+	} else {
+		if api.Spec.Path != "" {
+			contentPath = api.Spec.Path
+		}
 	}
-
 	body, err = yaml.Marshal(obj.Object)
 	if err != nil {
 		return
@@ -165,14 +205,11 @@ func HandleGitopsAPI(ctx context.Context, logger logr.Logger, git connectors.Con
 	if err != nil {
 		return
 	}
-
-	contentPath, err := text.Template(api.Spec.Path, obj.Object)
+	contentPath, err = text.Template(contentPath, obj.Object)
 	if err != nil {
 		return
 	}
-
 	logger.Info("Saving to", "path", contentPath, "kustomization", kustomizationPath, "object", title)
-
 	if err = copy(body, contentPath, fs, work); err != nil {
 		return
 	}

test/config.yaml

@@ -1,6 +1,4 @@
 name: kind-kind
-patches:
-  - ./test/patch1.yaml
 domain: 127.0.0.1.nip.io
 dex:
   disabled: true
@@ -20,25 +18,13 @@ podSubnet: 100.200.0.0/16
 serviceSubnet: 100.100.0.0/16
 calico:
   version: v3.8.2
-s3:
-  endpoint: http://minio.minio.svc:9000
-  externalEndpoint: minio.127.0.0.1.nip.io
-  bucket: harbor
-  access_key: minio
-  secret_key: minio123
-  region: us-east1
-  kmsMasterKey: minio-demo-key:6368616e676520746869732070617373776f726420746f206120736563726574
-  usePathStyle: true
-  skipTLSVerify: true
-  e2e:
-    minio: true
 ca:
-  cert: .certs/root-ca.crt
-  privateKey: .certs/root-ca.key
+  cert: ../.certs/root-ca.crt
+  privateKey: ../.certs/root-ca.key
   password: foobar
 ingressCA:
-  cert: .certs/ingress-ca.crt
-  privateKey: .certs/ingress-ca.key
+  cert: ../.certs/ingress-ca.crt
+  privateKey: ../.certs/ingress-ca.key
   password: foobar
 test:
   exclude:

test/e2e.go

@@ -46,11 +46,12 @@ var (
 	k8s    *kubernetes.Clientset
 	crdK8s crdclient.Client
 	tests  = map[string]Test{
-		"git-operator-is-running": TestGitOperatorIsRunning,
-		"github-branch-sync":      TestGithubBranchSync,
-		"github-pr-github-sync":   TestGithubPRSync,
-		"github-pr-crd-sync":      TestGithubPRCRDSync,
-		"github-gitops-api":       TestGitopsAPI,
+		"git-operator-is-running":       TestGitOperatorIsRunning,
+		"github-branch-sync":            TestGithubBranchSync,
+		"github-pr-github-sync":         TestGithubPRSync,
+		"github-pr-crd-sync":            TestGithubPRCRDSync,
+		"github-gitops-api":             TestGitopsAPI,
+		"github-gitops-api-search-path": TestGitopsAPISearchPath,
 	}
 	scheme              = runtime.NewScheme()
 	log                 = ctrl.Log.WithName("e2e")
@@ -85,7 +86,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	timeout = flag.Duration("timeout", 10*time.Minute, "Global timeout for all tests")
+	timeout = flag.Duration("timeout", 15*time.Minute, "Global timeout for all tests")
 	flag.Parse()
 
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
@@ -181,6 +182,50 @@ func TestGitopsAPI(ctx context.Context, test *console.TestResults) error {
 	return err
 }
 
+func TestGitopsAPISearchPath(ctx context.Context, test *console.TestResults) error {
+	git, err := connectors.NewConnector(ctx, crdK8s, k8s, log, "platform-system", "https://github.com/"+repository, &v1.LocalObjectReference{
+		Name: "github",
+	})
+	if err != nil {
+		return err
+	}
+	branchName := getBranchName("test")
+	body := `
+	{
+		"apiVersion": "v1",
+    	"data": {
+        	"some-key": "some-value",
+			"new-key": "new-value"
+    	},
+		"kind": "ConfigMap",
+    	"metadata": {
+        	"name": "test-configmap",
+        	"namespace": "default"
+		}
+	}
+	`
+
+	log.Info("json", "value", body)
+	_, pr, err := controllers.HandleGitopsAPI(ctx, log, git, gitv1.GitopsAPI{
+		Spec: gitv1.GitopsAPISpec{
+			GitRepository: repository,
+			Branch:        branchName,
+			SearchPath:    "resources/",
+			PullRequest: &gitv1.PullRequestTemplate{
+				Title: "New Automated PR - {{.metadata.name}}",
+				Body:  "Somebody created a new PR {{.metadata.name}}",
+			},
+		},
+	}, bytes.NewReader([]byte(body)))
+
+	if pr != 0 {
+		if err := git.ClosePullRequest(ctx, pr); err != nil {
+			return err
+		}
+	}
+	return err
+}
+
 func TestGitOperatorIsRunning(ctx context.Context, test *console.TestResults) error {
 	pods, err := k8s.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "control-plane=git-operator"})
 	if err != nil {
@@ -206,7 +251,7 @@ func TestGithubBranchSync(ctx context.Context, test *console.TestResults) error
 	}
 	test.Passf("TestGithubBranchSync", "Successfully created branch %s", branchName)
 
-	gitBranchGetCtx, cancelFunc := context.WithTimeout(ctx, 2*time.Minute)
+	gitBranchGetCtx, cancelFunc := context.WithTimeout(ctx, 3*time.Minute)
 	defer cancelFunc()
 	crdName := fmt.Sprintf("gitrepository-sample-%s", branchName)
 	gitBranch, err := waitForGitBranch(gitBranchGetCtx, crdName)
@@ -268,7 +313,7 @@ func TestGithubPRSync(ctx context.Context, test *console.TestResults) error {
 		return err
 	}
 
-	gitPRGetCtx, cancelFunc := context.WithTimeout(ctx, 2*time.Minute)
+	gitPRGetCtx, cancelFunc := context.WithTimeout(ctx, 3*time.Minute)
 	defer cancelFunc()
 	crdName := fmt.Sprintf("gitrepository-sample-%d", *pr.Number)
 	gitPR, err := waitForGitPullRequest(gitPRGetCtx, crdName)
@@ -363,7 +408,7 @@ func TestGithubPRCRDSync(ctx context.Context, test *console.TestResults) error {
 		return err
 	}
 
-	gitPRGetCtx, cancelFunc := context.WithTimeout(ctx, 2*time.Minute)
+	gitPRGetCtx, cancelFunc := context.WithTimeout(ctx, 3*time.Minute)
 	defer cancelFunc()
 	gitPR, err := waitForGitPullRequestFromCrd(gitPRGetCtx, branchName)
 	if err != nil {

test/e2e.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-export KARINA_VERSION=v0.16.2
+export KARINA_VERSION=v0.50.1
 export KARINA="./karina -c test/config.yaml"
 export KUBECONFIG=~/.kube/config
 export DOCKER_API_VERSION=1.39
@@ -30,14 +30,9 @@ export PATH=$(pwd)/.bin:$PATH
 
 $KARINA ca generate --name root-ca --cert-path .certs/root-ca.crt --private-key-path .certs/root-ca.key --password foobar  --expiry 1
 $KARINA ca generate --name ingress-ca --cert-path .certs/ingress-ca.crt --private-key-path .certs/ingress-ca.key --password foobar  --expiry 1
-$KARINA provision kind-cluster
-
-$KARINA deploy calico
-kubectl -n kube-system set env daemonset/calico-node FELIX_IGNORELOOSERPF=true
 
-$KARINA deploy base
-$KARINA deploy stubs
-kubectl create namespace platform-system
+$KARINA provision kind-cluster
+$KARINA deploy phases --bootstrap --stubs
 
 export IMG=flanksource/git-operator:v1
 make docker-build