Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/orchestrator/cmd/show-build-diff/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func main() {
fmt.Printf("Storage path %s\n", diffSource)
fmt.Printf("========\n")

onlyDiffMappings := make([]*header.BuildMap, 0)
onlyDiffMappings := make([]header.BuildMap, 0)

for _, mapping := range diffHeader.Mapping {
if mapping.BuildId == diffHeader.Metadata.BuildId {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func NewZeroDevice(size int64, blockSize int64) (*ZeroDevice, error) {
uint64(blockSize),
uint64(size),
),
[]*header.BuildMap{
[]header.BuildMap{
{
Offset: 0,
Length: uint64(size),
Expand Down
9 changes: 4 additions & 5 deletions packages/shared/pkg/storage/header/header.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,16 @@ const NormalizeFixVersion = 3

type Header struct {
Metadata *Metadata
Mapping []*BuildMap
Mapping []BuildMap
}

func NewHeader(metadata *Metadata, mapping []*BuildMap) (*Header, error) {
func NewHeader(metadata *Metadata, mapping []BuildMap) (*Header, error) {
if metadata.BlockSize == 0 {
return nil, fmt.Errorf("block size cannot be zero")
}

if len(mapping) == 0 {
mapping = []*BuildMap{{
mapping = []BuildMap{{
Offset: 0,
Length: metadata.Size,
BuildId: metadata.BuildId,
Expand Down Expand Up @@ -69,7 +69,6 @@ func (t *Header) GetShiftedMapping(ctx context.Context, offset int64) (mappedOff
return mappedOffset, mappedLength, buildID, nil
}

// TODO: Maybe we can optimize mapping by automatically assuming the mapping is uuid.Nil if we don't find it + stopping storing the nil mapping.
func (t *Header) getMapping(ctx context.Context, offset int64) (*BuildMap, int64, error) {
Comment thread
levb marked this conversation as resolved.
if offset < 0 || offset >= int64(t.Metadata.Size) {
if t.IsNormalizeFixApplied() {
Expand Down Expand Up @@ -102,7 +101,7 @@ func (t *Header) getMapping(ctx context.Context, offset int64) (*BuildMap, int64
return nil, 0, fmt.Errorf("no source found for offset %d", offset)
}

mapping := t.Mapping[i-1]
mapping := &t.Mapping[i-1]
shift := offset - int64(mapping.Offset)

// Verify that the offset falls within this mapping's range
Expand Down
12 changes: 6 additions & 6 deletions packages/shared/pkg/storage/header/inspect.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
// startBlock-endBlock [offset, offset+length) := [buildStorageOffset, buildStorageOffset+length) ⊂ buildId, length in bytes
//
// It is used for debugging and visualization.
func (mapping *BuildMap) Format(blockSize uint64) string {
func (mapping BuildMap) Format(blockSize uint64) string {
rangeMessage := fmt.Sprintf("%d-%d", mapping.Offset/blockSize, (mapping.Offset+mapping.Length)/blockSize)

return fmt.Sprintf(
Expand All @@ -30,7 +30,7 @@ const (
)

// Layers returns a map of buildIds that are present in the mappings.
func Layers(mappings []*BuildMap) *map[uuid.UUID]struct{} {
func Layers(mappings []BuildMap) *map[uuid.UUID]struct{} {
layers := make(map[uuid.UUID]struct{})

for _, mapping := range mappings {
Expand All @@ -44,7 +44,7 @@ func Layers(mappings []*BuildMap) *map[uuid.UUID]struct{} {
// It is used for debugging and visualization.
//
// You can pass maps to visualize different groups of buildIds.
func Visualize(mappings []*BuildMap, size, blockSize, cols uint64, bottomGroup, topGroup *map[uuid.UUID]struct{}) string {
func Visualize(mappings []BuildMap, size, blockSize, cols uint64, bottomGroup, topGroup *map[uuid.UUID]struct{}) string {
output := make([]rune, size/blockSize)

for outputIdx := range output {
Expand Down Expand Up @@ -85,7 +85,7 @@ func Visualize(mappings []*BuildMap, size, blockSize, cols uint64, bottomGroup,
//
// It checks if the mappings are contiguous and if the length of each mapping is a multiple of the block size.
// It also checks if the mappings cover the whole size.
func ValidateMappings(mappings []*BuildMap, size, blockSize uint64) error {
func ValidateMappings(mappings []BuildMap, size, blockSize uint64) error {
var currentOffset uint64

for _, mapping := range mappings {
Expand All @@ -111,11 +111,11 @@ func ValidateMappings(mappings []*BuildMap, size, blockSize uint64) error {
return nil
}

func (mapping *BuildMap) Equal(other *BuildMap) bool {
func (mapping BuildMap) Equal(other BuildMap) bool {
return mapping.Offset == other.Offset && mapping.Length == other.Length && mapping.BuildId == other.BuildId
}

func Equal(a, b []*BuildMap) bool {
func Equal(a, b []BuildMap) bool {
if len(a) != len(b) {
return false
}
Expand Down
69 changes: 24 additions & 45 deletions packages/shared/pkg/storage/header/mapping.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,12 @@ type BuildMap struct {
BuildStorageOffset uint64
}

func (mapping *BuildMap) Copy() *BuildMap {
return &BuildMap{
Offset: mapping.Offset,
Length: mapping.Length,
BuildId: mapping.BuildId,
BuildStorageOffset: mapping.BuildStorageOffset,
}
}

func CreateMapping(
buildId *uuid.UUID,
dirty *bitset.BitSet,
blockSize int64,
) []*BuildMap {
var mappings []*BuildMap
) []BuildMap {
var mappings []BuildMap

var startBlock uint
var blockLength uint
Expand All @@ -47,7 +38,7 @@ func CreateMapping(
}

if blockLength > 0 {
m := &BuildMap{
m := BuildMap{
Offset: uint64(startBlock) * uint64(blockSize),
BuildId: *buildId,
Length: uint64(blockLength) * uint64(blockSize),
Expand All @@ -64,7 +55,7 @@ func CreateMapping(
}

if blockLength > 0 {
mappings = append(mappings, &BuildMap{
mappings = append(mappings, BuildMap{
Offset: uint64(startBlock) * uint64(blockSize),
BuildId: *buildId,
Length: uint64(blockLength) * uint64(blockSize),
Expand All @@ -82,26 +73,26 @@ func CreateMapping(
//
// It returns a new set of mappings that covers the whole size.
func MergeMappings(
baseMapping []*BuildMap,
diffMapping []*BuildMap,
) []*BuildMap {
baseMapping []BuildMap,
diffMapping []BuildMap,
) []BuildMap {
if len(diffMapping) == 0 {
return baseMapping
}

baseMappingCopy := make([]*BuildMap, len(baseMapping))
baseMappingCopy := make([]BuildMap, len(baseMapping))

copy(baseMappingCopy, baseMapping)

baseMapping = baseMappingCopy

mappings := make([]*BuildMap, 0)
mappings := make([]BuildMap, 0)

var baseIdx int
var diffIdx int

for baseIdx < len(baseMapping) && diffIdx < len(diffMapping) {
base := baseMapping[baseIdx]
base := &baseMapping[baseIdx]
diff := diffMapping[diffIdx]

if base.Length == 0 {
Expand All @@ -119,7 +110,7 @@ func MergeMappings(
// base is before diff and there is no overlap
// add base to the result, because it will not be overlapping by any diff
if base.Offset+base.Length <= diff.Offset {
mappings = append(mappings, base)
mappings = append(mappings, *base)

baseIdx++

Expand Down Expand Up @@ -153,15 +144,13 @@ func MergeMappings(
leftBaseLength := int64(diff.Offset) - int64(base.Offset)

if leftBaseLength > 0 {
leftBase := &BuildMap{
mappings = append(mappings, BuildMap{
Offset: base.Offset,
Length: uint64(leftBaseLength),
BuildId: base.BuildId,
// the build storage offset is the same as the base mapping
BuildStorageOffset: base.BuildStorageOffset,
}

mappings = append(mappings, leftBase)
})
}

mappings = append(mappings, diff)
Expand All @@ -172,14 +161,12 @@ func MergeMappings(
rightBaseLength := int64(base.Length) - rightBaseShift

if rightBaseLength > 0 {
rightBase := &BuildMap{
baseMapping[baseIdx] = BuildMap{
Offset: base.Offset + uint64(rightBaseShift),
Length: uint64(rightBaseLength),
BuildId: base.BuildId,
BuildStorageOffset: base.BuildStorageOffset + uint64(rightBaseShift),
}

baseMapping[baseIdx] = rightBase
} else {
baseIdx++
}
Expand All @@ -199,14 +186,12 @@ func MergeMappings(
rightBaseLength := int64(base.Length) - rightBaseShift

if rightBaseLength > 0 {
rightBase := &BuildMap{
baseMapping[baseIdx] = BuildMap{
Offset: base.Offset + uint64(rightBaseShift),
Length: uint64(rightBaseLength),
BuildId: base.BuildId,
BuildStorageOffset: base.BuildStorageOffset + uint64(rightBaseShift),
}

baseMapping[baseIdx] = rightBase
} else {
baseIdx++
}
Expand All @@ -220,14 +205,12 @@ func MergeMappings(
leftBaseLength := int64(diff.Offset) - int64(base.Offset)

if leftBaseLength > 0 {
leftBase := &BuildMap{
mappings = append(mappings, BuildMap{
Offset: base.Offset,
Length: uint64(leftBaseLength),
BuildId: base.BuildId,
BuildStorageOffset: base.BuildStorageOffset,
}

mappings = append(mappings, leftBase)
})
}

baseIdx++
Expand All @@ -245,29 +228,25 @@ func MergeMappings(
}

// NormalizeMappings joins adjacent mappings that have the same buildId.
func NormalizeMappings(mappings []*BuildMap) []*BuildMap {
func NormalizeMappings(mappings []BuildMap) []BuildMap {
if len(mappings) == 0 {
return nil
}

result := make([]*BuildMap, 0, len(mappings))
result := make([]BuildMap, 0, len(mappings))

// Start with a copy of the first mapping
current := mappings[0].Copy()
current := mappings[0]

for i := 1; i < len(mappings); i++ {
mp := mappings[i]
if mp.BuildId != current.BuildId {
// BuildId changed, add the current map to results and start a new one
result = append(result, current)
current = mp.Copy() // New copy
} else {
// Same BuildId, just add the length
if mp.BuildId == current.BuildId {
current.Length += mp.Length
} else {
result = append(result, current)
current = mp
}
}

// Add the last mapping
result = append(result, current)

return result
Expand Down
Loading
Loading