Skip to content

Commit 0720a83

Browse files
neolynxneolynx
committed
[debian] publish: support MultiDist toggle
1 parent dd1dc7f commit 0720a83

3 files changed

Lines changed: 220 additions & 0 deletions

File tree

‎api/publish.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,9 @@ func apiPublishUpdateSwitch(c *gin.Context) {
502502
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
503503
}
504504

505+
// Capture MultiDist before mutations to detect a false→true transition.
506+
prevMultiDist := published.MultiDist
507+
505508
// Apply field mutations on the freshly loaded object.
506509
if b.SkipContents != nil {
507510
published.SkipContents = *b.SkipContents
@@ -549,6 +552,15 @@ func apiPublishUpdateSwitch(c *gin.Context) {
549552
if err != nil {
550553
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
551554
}
555+
// When MultiDist is toggled, the old pool layout still has files that
556+
// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
557+
// Run a second pass over the previous layout to remove stale files.
558+
if prevMultiDist != published.MultiDist {
559+
err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
560+
if err != nil {
561+
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
562+
}
563+
}
552564
}
553565

554566
return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil
@@ -1148,6 +1160,9 @@ func apiPublishUpdate(c *gin.Context) {
11481160
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
11491161
}
11501162

1163+
// Capture MultiDist before mutations to detect a false→true transition.
1164+
prevMultiDist := published.MultiDist
1165+
11511166
// Apply field mutations on the freshly loaded object.
11521167
if b.SkipContents != nil {
11531168
published.SkipContents = *b.SkipContents
@@ -1184,6 +1199,15 @@ func apiPublishUpdate(c *gin.Context) {
11841199
if err != nil {
11851200
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
11861201
}
1202+
// When MultiDist is toggled, the old pool layout still has files that
1203+
// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
1204+
// Run a second pass over the previous layout to remove stale files.
1205+
if prevMultiDist != published.MultiDist {
1206+
err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
1207+
if err != nil {
1208+
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
1209+
}
1210+
}
11871211
}
11881212

11891213
return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil

‎deb/publish.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1540,6 +1540,52 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
15401540
return referencedFiles, nil
15411541
}
15421542

1543+
// CleanupAfterMultiDistToggle cleans up stale pool files left behind when the
1544+
// MultiDist flag is toggled on a published repository.
1545+
//
1546+
// - false→true: Publish() wrote packages into pool/<distribution>/<component>/
1547+
// but the old flat pool/<component>/ files were not removed because
1548+
// CleanupPrefixComponentFiles only scans the new MultiDist tree.
1549+
// A second pass with MultiDist=false cleans the legacy flat layout by
1550+
// reusing the existing orphan-detection logic (the repo is now MultiDist=true
1551+
// so it is excluded from the referenced-files scan, making its old pool
1552+
// entries appear orphaned).
1553+
//
1554+
// - true→false: Publish() wrote packages into pool/<component>/ but the old
1555+
// per-distribution pool/<distribution>/<component>/ directories were not
1556+
// removed. The orphan-detection approach cannot be used here because the
1557+
// repo's RefList still contains all packages (they just moved locations).
1558+
// Instead we directly remove each pool/<distribution>/<component>/ directory.
1559+
// This is safe because per-distribution pool dirs are exclusive to a single
1560+
// prefix+distribution combination — no other published repo can share them.
1561+
func (collection *PublishedRepoCollection) CleanupAfterMultiDistToggle(publishedStorageProvider aptly.PublishedStorageProvider,
1562+
published *PublishedRepo, prevMultiDist bool, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
1563+
if prevMultiDist == published.MultiDist {
1564+
return nil
1565+
}
1566+
1567+
if !prevMultiDist && published.MultiDist {
1568+
// false→true: use orphan-detection via the existing cleanup, but with
1569+
// MultiDist temporarily set to false so it scans the flat pool layout.
1570+
legacy := *published
1571+
legacy.MultiDist = false
1572+
return collection.CleanupPrefixComponentFiles(publishedStorageProvider, &legacy, cleanComponents, collectionFactory, progress)
1573+
}
1574+
1575+
// true→false: directly remove the per-distribution pool directories.
1576+
publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage)
1577+
for _, component := range cleanComponents {
1578+
poolDir := filepath.Join(published.Prefix, "pool", published.Distribution, component)
1579+
if err := publishedStorage.RemoveDirs(poolDir, progress); err != nil {
1580+
return err
1581+
}
1582+
}
1583+
// Remove the distribution-level pool dir if it is now empty.
1584+
distPoolDir := filepath.Join(published.Prefix, "pool", published.Distribution)
1585+
_ = publishedStorage.RemoveDirs(distPoolDir, progress)
1586+
return nil
1587+
}
1588+
15431589
// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
15441590
func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider,
15451591
published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {

‎system/t12_api/publish.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,156 @@ def check(self):
443443
self.check_not_exists("public/" + prefix + "dists/")
444444

445445

446+
class PublishUpdateAPIMultiDistToggle(APITest):
447+
"""
448+
POST /publish/:prefix with MultiDist=false, then PUT to enable MultiDist=true
449+
"""
450+
fixtureGpg = True
451+
452+
def check(self):
453+
repo_name = self.random_name()
454+
self.check_equal(self.post(
455+
"/api/repos", json={"Name": repo_name, "DefaultDistribution": "bookworm"}).status_code, 201)
456+
457+
d = self.random_name()
458+
self.check_equal(self.upload("/api/files/" + d,
459+
"libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc",
460+
"pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz",
461+
"pyspi-0.6.1-1.3.stripped.dsc").status_code, 200)
462+
463+
task = self.post_task("/api/repos/" + repo_name + "/file/" + d)
464+
self.check_task(task)
465+
466+
# Publish with MultiDist=false (default)
467+
prefix = self.random_name()
468+
task = self.post_task(
469+
"/api/publish/" + prefix,
470+
json={
471+
"Architectures": ["i386", "source"],
472+
"SourceKind": "local",
473+
"Sources": [{"Name": repo_name}],
474+
"Signing": DefaultSigningOptions,
475+
"MultiDist": False,
476+
}
477+
)
478+
self.check_task(task)
479+
480+
repo_expected = {
481+
'AcquireByHash': False,
482+
'Architectures': ['i386', 'source'],
483+
'Codename': '',
484+
'Distribution': 'bookworm',
485+
'Label': '',
486+
'Origin': '',
487+
'NotAutomatic': '',
488+
'ButAutomaticUpgrades': '',
489+
'Path': prefix + '/' + 'bookworm',
490+
'Prefix': prefix,
491+
'SkipContents': False,
492+
'MultiDist': False,
493+
'SourceKind': 'local',
494+
'Sources': [{'Component': 'main', 'Name': repo_name}],
495+
'Storage': '',
496+
'Suite': ''}
497+
498+
all_repos = self.get("/api/publish")
499+
self.check_equal(all_repos.status_code, 200)
500+
self.check_in(repo_expected, all_repos.json())
501+
502+
# With MultiDist=false packages are stored under pool/main/...
503+
self.check_exists("public/" + prefix + "/dists/bookworm/Release")
504+
self.check_exists("public/" + prefix +
505+
"/dists/bookworm/main/binary-i386/Packages")
506+
self.check_exists("public/" + prefix +
507+
"/dists/bookworm/main/source/Sources")
508+
self.check_exists(
509+
"public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
510+
self.check_exists(
511+
"public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
512+
# MultiDist-style per-distribution pool must not exist yet
513+
self.check_not_exists(
514+
"public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
515+
516+
# Now update the published repo enabling MultiDist=true
517+
task = self.put_task(
518+
"/api/publish/" + prefix + "/bookworm",
519+
json={
520+
"MultiDist": True,
521+
"Signing": DefaultSigningOptions,
522+
}
523+
)
524+
self.check_task(task)
525+
526+
repo_expected_multidist = {
527+
'AcquireByHash': False,
528+
'Architectures': ['i386', 'source'],
529+
'Codename': '',
530+
'Distribution': 'bookworm',
531+
'Label': '',
532+
'Origin': '',
533+
'NotAutomatic': '',
534+
'ButAutomaticUpgrades': '',
535+
'Path': prefix + '/' + 'bookworm',
536+
'Prefix': prefix,
537+
'SkipContents': False,
538+
'MultiDist': True,
539+
'SourceKind': 'local',
540+
'Sources': [{'Component': 'main', 'Name': repo_name}],
541+
'Storage': '',
542+
'Suite': ''}
543+
544+
all_repos = self.get("/api/publish")
545+
self.check_equal(all_repos.status_code, 200)
546+
self.check_in(repo_expected_multidist, all_repos.json())
547+
548+
# After enabling MultiDist, packages are stored under pool/<distribution>/main/...
549+
self.check_exists("public/" + prefix + "/dists/bookworm/Release")
550+
self.check_exists("public/" + prefix +
551+
"/dists/bookworm/main/binary-i386/Packages")
552+
self.check_exists("public/" + prefix +
553+
"/dists/bookworm/main/source/Sources")
554+
self.check_exists(
555+
"public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
556+
self.check_exists(
557+
"public/" + prefix + "/pool/bookworm/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
558+
# Flat pool must not exist while MultiDist is on
559+
self.check_not_exists(
560+
"public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
561+
562+
# Switch MultiDist back to false
563+
task = self.put_task(
564+
"/api/publish/" + prefix + "/bookworm",
565+
json={
566+
"MultiDist": False,
567+
"Signing": DefaultSigningOptions,
568+
}
569+
)
570+
self.check_task(task)
571+
572+
repo_expected["MultiDist"] = False
573+
all_repos = self.get("/api/publish")
574+
self.check_equal(all_repos.status_code, 200)
575+
self.check_in(repo_expected, all_repos.json())
576+
577+
# Packages are back under the flat pool/main/...
578+
self.check_exists("public/" + prefix + "/dists/bookworm/Release")
579+
self.check_exists("public/" + prefix +
580+
"/dists/bookworm/main/binary-i386/Packages")
581+
self.check_exists("public/" + prefix +
582+
"/dists/bookworm/main/source/Sources")
583+
self.check_exists(
584+
"public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
585+
self.check_exists(
586+
"public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
587+
# Per-distribution pool must be gone
588+
self.check_not_exists(
589+
"public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
590+
591+
task = self.delete_task("/api/publish/" + prefix + "/bookworm")
592+
self.check_task(task)
593+
self.check_not_exists("public/" + prefix + "dists/")
594+
595+
446596
class PublishConcurrentUpdateAPITestRepo(APITest):
447597
"""
448598
PUT /publish/:prefix/:distribution (local repos), DELETE /publish/:prefix/:distribution

0 commit comments

Comments
 (0)