Skip to content

Commit fdfc139

Browse files
kushnaidukushnaidu
authored
Create db indexing for upstream ref (#1043)
* All upstream ref indexing in the DB Signed-off-by: Kushal Harish Naidu <kushal.harish.naidu@ericsson.com> * Add mock with maybe for flaky test Signed-off-by: Kushal Harish Naidu <kushal.harish.naidu@ericsson.com> * Update backfill code to interate rows with prepared statement Signed-off-by: Kushal Harish Naidu <kushal.harish.naidu@ericsson.com> * Add batching for backfill and add nil check for pr object Signed-off-by: Kushal Harish Naidu <kushal.harish.naidu@ericsson.com> * Remove offset and use pagination using primary key Signed-off-by: Kushal Harish Naidu <kushal.harish.naidu@ericsson.com> --------- Signed-off-by: Kushal Harish Naidu <kushal.harish.naidu@ericsson.com>
1 parent be2747b commit fdfc139

11 files changed

Lines changed: 392 additions & 92 deletions

api/sql/porch-db-1.5.10-1.5.9.sql renamed to api/sql/porch-db-1.5.11-1.5.9.sql

File renamed without changes.

‎api/sql/porch-db-1.5.11-1.6.0.sql‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
Copyright 2026 The kpt Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
-- Add the upstream_ref_name column to store the upstream package revision name
18+
-- extracted from tasks. This replaces the regex-based search on the tasks column
19+
-- with an indexed lookup for findUpstreamRefsFromDB.
20+
--
21+
-- Existing rows default to ''. The Porch server automatically backfills
22+
-- this column on startup by parsing the tasks JSON and extracting the
23+
-- upstream reference name from clone/upgrade tasks.
24+
-- No manual resync is required.
25+
ALTER TABLE package_revisions
26+
ADD COLUMN IF NOT EXISTS upstream_ref_name TEXT NOT NULL DEFAULT '';
27+
28+
-- Create a partial B-tree index for fast upstream reference lookups.
29+
-- Only indexes rows that have a non-empty upstream_ref_name and are not
30+
-- auto-managed main branch packages (revision != -1).
31+
CREATE INDEX IF NOT EXISTS idx_package_revisions_upstream_ref
32+
ON package_revisions (k8s_name_space, upstream_ref_name)
33+
WHERE upstream_ref_name != '' AND revision != -1;
34+
35+
-- Hot path: "Get the latest revision of a package" -- issued by every rpkg_approve.
36+
CREATE INDEX IF NOT EXISTS idx_package_revisions_pkg_revdesc
37+
ON package_revisions (k8s_name_space, package_k8s_name, revision DESC);
38+
39+
-- Hot path: "List PRs in lifecycle X" -- background reconcilers, dashboards,
40+
-- and DeletionProposed sweeps.
41+
CREATE INDEX IF NOT EXISTS idx_package_revisions_lifecycle
42+
ON package_revisions (lifecycle);
43+
44+
-- Hot path: "Find the latest=true revision of a package" -- partial index
45+
-- saves ~50% storage vs a full index since most revisions have latest=false.
46+
CREATE INDEX IF NOT EXISTS idx_package_revisions_latest_partial
47+
ON package_revisions (k8s_name_space, package_k8s_name)
48+
WHERE latest = true;
49+
50+
-- Hot path: "List all packages in a repository" -- per-repo reconciliation,
51+
-- runs every sync.schedule cycle.
52+
CREATE INDEX IF NOT EXISTS idx_packages_repo
53+
ON packages (k8s_name_space, repo_k8s_name);

api/sql/porch-db-1.5.9-1.5.10.sql renamed to api/sql/porch-db-1.5.9-1.5.11.sql

File renamed without changes.

‎api/sql/porch-db-1.5.9-1.6.0.sql‎

Lines changed: 0 additions & 37 deletions
This file was deleted.

api/sql/porch-db-1.6.0-1.5.9.sql renamed to api/sql/porch-db-1.6.0-1.5.11.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ limitations under the License.
1515
*/
1616

1717
ALTER TABLE package_revisions
18-
DROP COLUMN IF EXISTS resources_size;
18+
DROP COLUMN IF EXISTS upstream_ref_name;

‎api/sql/porch-db.sql‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,31 @@ CREATE TABLE IF NOT EXISTS package_revisions (
8787
tasks TEXT NOT NULL,
8888
kptfile_status TEXT NOT NULL DEFAULT '{}',
8989
resources_size BIGINT NOT NULL DEFAULT 0,
90+
upstream_ref_name TEXT NOT NULL DEFAULT '',
9091
PRIMARY KEY (k8s_name_space, k8s_name),
9192
CONSTRAINT fk_package
9293
FOREIGN KEY (k8s_name_space, package_k8s_name)
9394
REFERENCES packages (k8s_name_space, k8s_name)
9495
ON DELETE CASCADE
9596
);
9697

98+
CREATE INDEX IF NOT EXISTS idx_package_revisions_upstream_ref
99+
ON package_revisions (k8s_name_space, upstream_ref_name)
100+
WHERE upstream_ref_name != '' AND revision != -1;
101+
102+
CREATE INDEX IF NOT EXISTS idx_package_revisions_pkg_revdesc
103+
ON package_revisions (k8s_name_space, package_k8s_name, revision DESC);
104+
105+
CREATE INDEX IF NOT EXISTS idx_package_revisions_lifecycle
106+
ON package_revisions (lifecycle);
107+
108+
CREATE INDEX IF NOT EXISTS idx_package_revisions_latest_partial
109+
ON package_revisions (k8s_name_space, package_k8s_name)
110+
WHERE latest = true;
111+
112+
CREATE INDEX IF NOT EXISTS idx_packages_repo
113+
ON packages (k8s_name_space, repo_k8s_name);
114+
97115
CREATE OR REPLACE FUNCTION check_package_revisions_columns() RETURNS trigger
98116
LANGUAGE plpgsql AS
99117
$BODY$

‎deployments/porch/3-porch-postgres-bundle.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,13 +355,31 @@ data:
355355
tasks TEXT NOT NULL,
356356
kptfile_status TEXT NOT NULL DEFAULT '{}',
357357
resources_size BIGINT NOT NULL DEFAULT 0,
358+
upstream_ref_name TEXT NOT NULL DEFAULT '',
358359
PRIMARY KEY (k8s_name_space, k8s_name),
359360
CONSTRAINT fk_package
360361
FOREIGN KEY (k8s_name_space, package_k8s_name)
361362
REFERENCES packages (k8s_name_space, k8s_name)
362363
ON DELETE CASCADE
363364
);
364365
366+
CREATE INDEX IF NOT EXISTS idx_package_revisions_upstream_ref
367+
ON package_revisions (k8s_name_space, upstream_ref_name)
368+
WHERE upstream_ref_name != '' AND revision != -1;
369+
370+
CREATE INDEX IF NOT EXISTS idx_package_revisions_pkg_revdesc
371+
ON package_revisions (k8s_name_space, package_k8s_name, revision DESC);
372+
373+
CREATE INDEX IF NOT EXISTS idx_package_revisions_lifecycle
374+
ON package_revisions (lifecycle);
375+
376+
CREATE INDEX IF NOT EXISTS idx_package_revisions_latest_partial
377+
ON package_revisions (k8s_name_space, package_k8s_name)
378+
WHERE latest = true;
379+
380+
CREATE INDEX IF NOT EXISTS idx_packages_repo
381+
ON packages (k8s_name_space, repo_k8s_name);
382+
365383
CREATE OR REPLACE FUNCTION check_package_revisions_columns() RETURNS trigger
366384
LANGUAGE plpgsql AS
367385
$BODY$

‎pkg/cache/dbcache/dbcachefactory.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ func (f *DBCacheFactory) NewCache(ctx context.Context, options cachetypes.CacheO
4040
return nil, fmt.Errorf("kptfile_status backfill failed: %w", err)
4141
}
4242

43+
if err := backfillUpstreamRefName(ctx); err != nil {
44+
return nil, fmt.Errorf("upstream_ref_name backfill failed: %w", err)
45+
}
46+
4347
return &dbCache{
4448
repositories: repomap.SafeRepoMap{},
4549
options: options,

‎pkg/cache/dbcache/dbpackagerevision_test.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,3 +617,96 @@ status:
617617

618618
t.deleteTestRepo(dbRepo.Key())
619619
}
620+
621+
func (t *DbTestSuite) TestBackfillUpstreamRefName() {
622+
mockCache := mockcachetypes.NewMockCache(t.T())
623+
cachetypes.CacheInstance = mockCache
624+
mockCache.EXPECT().GetRepository(mock.Anything).Return(&dbRepository{})
625+
626+
dbRepo := t.createTestRepo("backfill-ns", "upstream-backfill-repo")
627+
dbPkg := t.createTestPkg(dbRepo.Key(), "downstream-pkg")
628+
629+
upstreamPRName := "upstream-backfill-repo.basepkg.v1"
630+
631+
// Write a PR with clone task — pkgRevWriteToDB will set upstream_ref_name automatically.
632+
pr := dbPackageRevision{
633+
pkgRevKey: repository.PackageRevisionKey{
634+
PkgKey: dbPkg.Key(),
635+
WorkspaceName: "ws-clone",
636+
Revision: 1,
637+
},
638+
lifecycle: "Published",
639+
tasks: []porchapi.Task{
640+
{
641+
Type: porchapi.TaskTypeClone,
642+
Clone: &porchapi.PackageCloneTaskSpec{
643+
Upstream: porchapi.UpstreamPackage{
644+
UpstreamRef: &porchapi.PackageRevisionRef{Name: upstreamPRName},
645+
},
646+
},
647+
},
648+
},
649+
}
650+
t.Require().NoError(pkgRevWriteToDB(t.Context(), &pr))
651+
652+
// Simulate a pre-migration row by clearing the upstream_ref_name column directly.
653+
_, err := GetDB().db.Exec(t.Context(),
654+
`UPDATE package_revisions SET upstream_ref_name = '' WHERE k8s_name_space = $1 AND k8s_name = $2`,
655+
pr.Key().K8SNS(), pr.Key().K8SName())
656+
t.Require().NoError(err)
657+
658+
// Verify that findUpstreamRefsFromDB finds nothing before backfill.
659+
found, err := findUpstreamRefsFromDB(t.Context(), "backfill-ns", upstreamPRName)
660+
t.Require().NoError(err)
661+
t.Empty(found)
662+
663+
// Run backfill.
664+
err = backfillUpstreamRefName(t.Context())
665+
t.Require().NoError(err)
666+
667+
// Verify upstream_ref_name is now populated.
668+
found, err = findUpstreamRefsFromDB(t.Context(), "backfill-ns", upstreamPRName)
669+
t.Require().NoError(err)
670+
t.Equal(pr.Key().K8SName(), found)
671+
672+
// Run backfill again — should be a no-op.
673+
err = backfillUpstreamRefName(t.Context())
674+
t.Require().NoError(err)
675+
676+
// Also test with an upgrade task.
677+
newUpstreamName := "upstream-backfill-repo.basepkg.v2"
678+
pr2 := dbPackageRevision{
679+
pkgRevKey: repository.PackageRevisionKey{
680+
PkgKey: dbPkg.Key(),
681+
WorkspaceName: "ws-upgrade",
682+
Revision: 2,
683+
},
684+
lifecycle: "Published",
685+
tasks: []porchapi.Task{
686+
{
687+
Type: porchapi.TaskTypeUpgrade,
688+
Upgrade: &porchapi.PackageUpgradeTaskSpec{
689+
NewUpstream: porchapi.PackageRevisionRef{Name: newUpstreamName},
690+
},
691+
},
692+
},
693+
}
694+
t.Require().NoError(pkgRevWriteToDB(t.Context(), &pr2))
695+
696+
// Clear upstream_ref_name to simulate pre-migration state.
697+
_, err = GetDB().db.Exec(t.Context(),
698+
`UPDATE package_revisions SET upstream_ref_name = '' WHERE k8s_name_space = $1 AND k8s_name = $2`,
699+
pr2.Key().K8SNS(), pr2.Key().K8SName())
700+
t.Require().NoError(err)
701+
702+
// Run backfill.
703+
err = backfillUpstreamRefName(t.Context())
704+
t.Require().NoError(err)
705+
706+
// Verify.
707+
found, err = findUpstreamRefsFromDB(t.Context(), "backfill-ns", newUpstreamName)
708+
t.Require().NoError(err)
709+
t.Equal(pr2.Key().K8SName(), found)
710+
711+
t.deleteTestRepo(dbRepo.Key())
712+
}

0 commit comments

Comments
 (0)