manifest: move L0-specific fields from TableMetadata into l0Sublevels

TableMetadata contained four L0-specific fields (SubLevel, L0Index,
minIntervalIndex, maxIntervalIndex) that were only meaningful for L0
files, only valid for the most recent Version's l0Sublevels, and written
exclusively by l0Sublevels code. Storing them on the shared TableMetadata
struct was fragile because any code could accidentally read stale values
from a non-current Version.

This change introduces an l0FileState struct and stores per-file state in
l0Sublevels via a fileState slice (indexed by file position) and a
fileStateMap (TableNum -> index). The L0Index concept is now implicit as
the index into fileState.

L0CompactionFiles.FilesIncluded is changed from bitSet (indexed by
L0Index) to map[base.TableNum]struct{}, eliminating the need for
L0Index to be accessible outside l0Sublevels.

An L0Organizer.SubLevelOf method is added for external callers (e.g.
generateSublevelInfo in compaction_picker.go) that need to look up a
file's sublevel.

Co-Authored-By: roachdev-claude <roachdev-claude-bot@cockroachlabs.com>

Author: RaduBerinde

compaction_picker.go

@@ -143,10 +143,13 @@ func (cl sublevelInfo) String() string {
 
 // generateSublevelInfo will generate the level slices for each of the sublevels
 // from the level slice for all of L0.
-func generateSublevelInfo(cmp base.Compare, levelFiles manifest.LevelSlice) []sublevelInfo {
+func generateSublevelInfo(
+	cmp base.Compare, levelFiles manifest.LevelSlice, l0Organizer *manifest.L0Organizer,
+) []sublevelInfo {
 	sublevelMap := make(map[uint64][]*manifest.TableMetadata)
 	for f := range levelFiles.All() {
-		sublevelMap[uint64(f.SubLevel)] = append(sublevelMap[uint64(f.SubLevel)], f)
+		sl := uint64(l0Organizer.SubLevelOf(f))
+		sublevelMap[sl] = append(sublevelMap[sl], f)
 	}
 
 	var sublevels []int
@@ -442,7 +445,7 @@ func (pc *pickedTableCompaction) setupInputs(
 
 	if inputLevel.level == 0 {
 		// If L0 is involved, it should always be the startLevel of the compaction.
-		pc.startLevel.l0SublevelInfo = generateSublevelInfo(cmp, pc.startLevel.files)
+		pc.startLevel.l0SublevelInfo = generateSublevelInfo(cmp, pc.startLevel.files, pc.l0Organizer)
 	}
 
 	if outputKeyRangeAlreadyCompacting(cmp, inProgressCompactions, pc) {
@@ -546,7 +549,7 @@ func (pc *pickedTableCompaction) maybeGrowL0ForBase(cmp base.Compare, maxExpande
 	iter := pc.version.Levels[0].Iter()
 	var sizeSum uint64
 	for j, f := 0, iter.First(); f != nil; j, f = j+1, iter.Next() {
-		if pc.lcf.FilesIncluded[f.L0Index] {
+		if _, ok := pc.lcf.FilesIncluded[f.TableNum]; ok {
 			newStartLevelFiles = append(newStartLevelFiles, f)
 			sizeSum += f.Size
 		}

compaction_picker_test.go

@@ -1075,6 +1075,11 @@ func TestPickedCompactionSetupInputs(t *testing.T) {
 				pc.outputLevel.level = pc.startLevel.level + 1
 			}
 			pc.version = newVersion(opts, files)
+			if pc.startLevel.level == 0 {
+				l0Organizer := manifest.NewL0Organizer(opts.Comparer, opts.FlushSplitBytes)
+				l0Organizer.ResetForTesting(pc.version)
+				pc.l0Organizer = l0Organizer
+			}
 			pc.startLevel.files = pc.version.Overlaps(
 				pc.startLevel.level,
 				base.UserKeyBoundsInclusive([]byte(args[0].String()), []byte(args[1].String())),

compaction_test.go

@@ -2368,7 +2368,6 @@ func TestCompactionCheckOrdering(t *testing.T) {
 				var sublevels []manifest.LevelSlice
 				var files *[]*manifest.TableMetadata
 				var sublevel []*manifest.TableMetadata
-				var sublevelNum int
 				var parsingSublevel bool
 				tableNum := base.TableNum(1)
 
@@ -2390,10 +2389,14 @@ func TestCompactionCheckOrdering(t *testing.T) {
 						if err != nil {
 							return err.Error()
 						}
-						sublevelNum, err = strconv.Atoi(data[3:])
+						sublevelNum, err := strconv.Atoi(data[3:])
 						if err != nil {
 							return err.Error()
 						}
+						if sublevelNum != len(sublevels) {
+							d.Fatalf(t, "L0 sublevels should be specified in order; expected L0.%d, got L0.%d", len(sublevels), sublevelNum)
+						}
+
 						if c.startLevel.level == -1 {
 							c.startLevel.level = level
 							files = &startFiles
@@ -2423,7 +2426,6 @@ func TestCompactionCheckOrdering(t *testing.T) {
 						tableNum++
 						*files = append(*files, meta)
 						if parsingSublevel {
-							meta.SubLevel = sublevelNum
 							sublevel = append(sublevel, meta)
 						}
 					}
@@ -2437,7 +2439,21 @@ func TestCompactionCheckOrdering(t *testing.T) {
 				}
 				if c.startLevel.level == 0 {
 					// We don't change the input files for the compaction beyond this point.
-					c.startLevel.l0SublevelInfo = generateSublevelInfo(c.comparer.Compare, c.startLevel.files)
+					// Build l0SublevelInfo directly from the explicitly-specified sublevels.
+					if len(sublevels) > 0 {
+						for i, sl := range sublevels {
+							c.startLevel.l0SublevelInfo = append(c.startLevel.l0SublevelInfo, sublevelInfo{
+								LevelSlice: sl,
+								sublevel:   manifest.L0Sublevel(i),
+							})
+						}
+					} else {
+						// No explicit sublevels; put all files in sublevel 0.
+						c.startLevel.l0SublevelInfo = []sublevelInfo{{
+							LevelSlice: manifest.NewLevelSliceKeySorted(c.comparer.Compare, startFiles),
+							sublevel:   manifest.L0Sublevel(0),
+						}}
+					}
 				}
 
 				newIters := func(

internal/compact/testdata/tombstone_elision_setup

@@ -144,21 +144,21 @@ L0 [a, z]: [a, j) [k, z)
 define
 L0
   1:[a#1,SET-b#inf,RANGEDEL]
-  1:[c#1,SET-d#inf,RANGEDEL]
-  1:[f#1,SET-g#inf,RANGEDEL]
-  1:[i#1,SET-j#inf,RANGEDEL]
+  2:[c#1,SET-d#inf,RANGEDEL]
+  3:[f#1,SET-g#inf,RANGEDEL]
+  4:[i#1,SET-j#inf,RANGEDEL]
 L6
-  1:[a#0,SET-i#inf,RANGEDEL]
-  1:[k#0,SET-z#inf,RANGEDEL]
+  5:[a#0,SET-i#inf,RANGEDEL]
+  6:[k#0,SET-z#inf,RANGEDEL]
 ----
 L0.0:
   000001:[a#1,SET-b#inf,RANGEDEL]
-  000001:[c#1,SET-d#inf,RANGEDEL]
-  000001:[f#1,SET-g#inf,RANGEDEL]
-  000001:[i#1,SET-j#inf,RANGEDEL]
+  000002:[c#1,SET-d#inf,RANGEDEL]
+  000003:[f#1,SET-g#inf,RANGEDEL]
+  000004:[i#1,SET-j#inf,RANGEDEL]
 L6:
-  000001:[a#0,SET-i#inf,RANGEDEL]
-  000001:[k#0,SET-z#inf,RANGEDEL]
+  000005:[a#0,SET-i#inf,RANGEDEL]
+  000006:[k#0,SET-z#inf,RANGEDEL]
 
 # L0 compaction should reflect the ranges in L0 as well.
 inuse-key-ranges