Skip to content

Commit 3a4e40b

Browse files
elliotchenzichangelliotchenzichang
authored
Merge pull request #19 from elliotchenzichang/optimize-current-project-by-cursor
Optimize current project by cursor
2 parents 404833f + 617b36d commit 3a4e40b

2 files changed

Lines changed: 341 additions & 0 deletions

File tree

‎storage/hint.go‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package storage
2+
3+
import (
4+
"encoding/binary"
5+
"errors"
6+
"fmt"
7+
"io"
8+
"os"
9+
10+
"tiny-bitcask/entity"
11+
)
12+
13+
const (
14+
hintMagic = "TBHK"
15+
hintVersion = byte(1)
16+
hintHeaderLen = 8
17+
)
18+
19+
var (
20+
ErrInvalidHintFile = errors.New("storage: invalid or unsupported hint file")
21+
)
22+
23+
// HintRecord is one row in a .hint file (compact keydir metadata for a segment).
24+
type HintRecord struct {
25+
Timestamp uint64
26+
KeySize uint32
27+
ValueSize uint32
28+
RecordOffset int64
29+
Flag uint8
30+
Key []byte
31+
}
32+
33+
// HintFilePath returns the path to the hint file for segment fid.
34+
func HintFilePath(dir string, fid int) string {
35+
return fmt.Sprintf("%s/%d.hint", dir, fid)
36+
}
37+
38+
// WriteHintFileForDataFile scans a sealed .dat file and writes a companion .hint file
39+
// (timestamp, sizes, record offset, flag, key only — no values).
40+
func WriteHintFileForDataFile(dir string, fid int, verifyCRC bool) error {
41+
datPath := getFilePath(dir, fid)
42+
of, err := NewOldFile(datPath, verifyCRC)
43+
if err != nil {
44+
return err
45+
}
46+
defer of.Close()
47+
48+
tmpPath := HintFilePath(dir, fid) + ".tmp"
49+
f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
50+
if err != nil {
51+
return err
52+
}
53+
54+
header := make([]byte, hintHeaderLen)
55+
copy(header[0:4], hintMagic)
56+
header[4] = hintVersion
57+
if _, err := f.Write(header); err != nil {
58+
f.Close()
59+
os.Remove(tmpPath)
60+
return err
61+
}
62+
63+
var off int64
64+
for {
65+
entry, err := of.ReadEntityWithOutLength(off)
66+
if err != nil {
67+
if err == io.EOF {
68+
break
69+
}
70+
f.Close()
71+
os.Remove(tmpPath)
72+
return err
73+
}
74+
recOff := off
75+
off += entry.Size()
76+
77+
if entry.Meta.Flag == entity.DeleteFlag {
78+
continue
79+
}
80+
81+
rec := make([]byte, 25+len(entry.Key))
82+
binary.LittleEndian.PutUint64(rec[0:8], entry.Meta.TimeStamp)
83+
binary.LittleEndian.PutUint32(rec[8:12], entry.Meta.KeySize)
84+
binary.LittleEndian.PutUint32(rec[12:16], entry.Meta.ValueSize)
85+
binary.LittleEndian.PutUint64(rec[16:24], uint64(recOff))
86+
rec[24] = entry.Meta.Flag
87+
copy(rec[25:], entry.Key)
88+
if _, err := f.Write(rec); err != nil {
89+
f.Close()
90+
os.Remove(tmpPath)
91+
return err
92+
}
93+
}
94+
95+
if err := f.Sync(); err != nil {
96+
f.Close()
97+
os.Remove(tmpPath)
98+
return err
99+
}
100+
if err := f.Close(); err != nil {
101+
os.Remove(tmpPath)
102+
return err
103+
}
104+
105+
hintPath := HintFilePath(dir, fid)
106+
if err := os.Rename(tmpPath, hintPath); err != nil {
107+
os.Remove(tmpPath)
108+
return err
109+
}
110+
return nil
111+
}
112+
113+
// ReadHintFile reads and parses a .hint file. Caller must validate it matches the .dat.
114+
func ReadHintFile(dir string, fid int) ([]HintRecord, error) {
115+
p := HintFilePath(dir, fid)
116+
f, err := os.Open(p)
117+
if err != nil {
118+
return nil, err
119+
}
120+
defer f.Close()
121+
122+
st, err := f.Stat()
123+
if err != nil {
124+
return nil, err
125+
}
126+
if st.Size() < int64(hintHeaderLen) {
127+
return nil, ErrInvalidHintFile
128+
}
129+
130+
header := make([]byte, hintHeaderLen)
131+
if _, err := io.ReadFull(f, header); err != nil {
132+
return nil, err
133+
}
134+
if string(header[0:4]) != hintMagic || header[4] != hintVersion {
135+
return nil, ErrInvalidHintFile
136+
}
137+
138+
var out []HintRecord
139+
for {
140+
fixed := make([]byte, 25)
141+
_, err := io.ReadFull(f, fixed)
142+
if err == io.EOF {
143+
break
144+
}
145+
if err != nil {
146+
if err == io.ErrUnexpectedEOF {
147+
return nil, ErrInvalidHintFile
148+
}
149+
return nil, err
150+
}
151+
ts := binary.LittleEndian.Uint64(fixed[0:8])
152+
ks := binary.LittleEndian.Uint32(fixed[8:12])
153+
vs := binary.LittleEndian.Uint32(fixed[12:16])
154+
recOff := int64(binary.LittleEndian.Uint64(fixed[16:24]))
155+
flag := fixed[24]
156+
157+
key := make([]byte, ks)
158+
if _, err := io.ReadFull(f, key); err != nil {
159+
return nil, ErrInvalidHintFile
160+
}
161+
out = append(out, HintRecord{
162+
Timestamp: ts,
163+
KeySize: ks,
164+
ValueSize: vs,
165+
RecordOffset: recOff,
166+
Flag: flag,
167+
Key: key,
168+
})
169+
}
170+
return out, nil
171+
}
172+
173+
// HintFileExists reports whether a hint file is present for the segment.
174+
func HintFileExists(dir string, fid int) bool {
175+
st, err := os.Stat(HintFilePath(dir, fid))
176+
return err == nil && !st.IsDir()
177+
}
178+
179+
// RemoveHintFile removes the hint file for a segment if it exists.
180+
func RemoveHintFile(dir string, fid int) {
181+
_ = os.Remove(HintFilePath(dir, fid))
182+
}

‎storage/hint_test.go‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package storage
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
"tiny-bitcask/entity"
11+
)
12+
13+
func TestHintFile_WriteReadRoundtrip(t *testing.T) {
14+
tests := []struct {
15+
name string
16+
key []byte
17+
value []byte
18+
wantKeySize uint32
19+
wantValueSize uint32
20+
wantOffset int64
21+
}{
22+
{
23+
name: "ascii_key_value",
24+
key: []byte("hello"),
25+
value: []byte("world"),
26+
wantKeySize: 5,
27+
wantValueSize: 5,
28+
wantOffset: 0,
29+
},
30+
{
31+
name: "empty_value",
32+
key: []byte("k"),
33+
value: []byte{},
34+
wantKeySize: 1,
35+
wantValueSize: 0,
36+
wantOffset: 0,
37+
},
38+
}
39+
for _, tt := range tests {
40+
t.Run(tt.name, func(t *testing.T) {
41+
dir := t.TempDir()
42+
fid := 1
43+
datPath := getFilePath(dir, fid)
44+
f, err := os.OpenFile(datPath, os.O_CREATE|os.O_RDWR, 0o644)
45+
require.NoError(t, err)
46+
47+
e := entity.NewEntryWithData(tt.key, tt.value)
48+
buf := e.Encode()
49+
_, err = f.WriteAt(buf, 0)
50+
require.NoError(t, err)
51+
require.NoError(t, f.Close())
52+
53+
require.NoError(t, WriteHintFileForDataFile(dir, fid, true))
54+
55+
assert.True(t, HintFileExists(dir, fid))
56+
57+
recs, err := ReadHintFile(dir, fid)
58+
require.NoError(t, err)
59+
require.Len(t, recs, 1)
60+
r := recs[0]
61+
assert.Equal(t, e.Meta.TimeStamp, r.Timestamp)
62+
assert.Equal(t, tt.wantKeySize, r.KeySize)
63+
assert.Equal(t, tt.wantValueSize, r.ValueSize)
64+
assert.Equal(t, tt.wantOffset, r.RecordOffset)
65+
assert.Equal(t, byte(0), r.Flag)
66+
assert.Equal(t, string(tt.key), string(r.Key))
67+
})
68+
}
69+
}
70+
71+
func TestHintFile_SkipsTombstone(t *testing.T) {
72+
tests := []struct {
73+
name string
74+
setup func(t *testing.T, f *os.File) error
75+
wantLen int
76+
wantKey string
77+
}{
78+
{
79+
name: "live_then_tombstone",
80+
setup: func(t *testing.T, f *os.File) error {
81+
t.Helper()
82+
live := entity.NewEntryWithData([]byte("k1"), []byte("v1"))
83+
off := int64(0)
84+
if _, err := f.WriteAt(live.Encode(), off); err != nil {
85+
return err
86+
}
87+
off += live.Size()
88+
tomb := entity.NewTombstoneEntry([]byte("k2"))
89+
_, err := f.WriteAt(tomb.Encode(), off)
90+
return err
91+
},
92+
wantLen: 1,
93+
wantKey: "k1",
94+
},
95+
{
96+
name: "only_tombstone",
97+
setup: func(t *testing.T, f *os.File) error {
98+
t.Helper()
99+
tomb := entity.NewTombstoneEntry([]byte("k2"))
100+
_, err := f.WriteAt(tomb.Encode(), 0)
101+
return err
102+
},
103+
wantLen: 0,
104+
wantKey: "",
105+
},
106+
}
107+
for _, tt := range tests {
108+
t.Run(tt.name, func(t *testing.T) {
109+
dir := t.TempDir()
110+
fid := 2
111+
datPath := getFilePath(dir, fid)
112+
f, err := os.OpenFile(datPath, os.O_CREATE|os.O_RDWR, 0o644)
113+
require.NoError(t, err)
114+
require.NoError(t, tt.setup(t, f))
115+
require.NoError(t, f.Close())
116+
117+
require.NoError(t, WriteHintFileForDataFile(dir, fid, true))
118+
recs, err := ReadHintFile(dir, fid)
119+
require.NoError(t, err)
120+
require.Len(t, recs, tt.wantLen)
121+
if tt.wantLen > 0 {
122+
assert.Equal(t, tt.wantKey, string(recs[0].Key))
123+
}
124+
})
125+
}
126+
}
127+
128+
func TestHintFile_InvalidHeader(t *testing.T) {
129+
tests := []struct {
130+
name string
131+
fid int
132+
content []byte
133+
}{
134+
{
135+
name: "wrong_magic",
136+
fid: 9,
137+
content: []byte("BAD!"),
138+
},
139+
{
140+
name: "too_short",
141+
fid: 10,
142+
content: []byte("TBH"),
143+
},
144+
{
145+
name: "bad_version",
146+
fid: 11,
147+
content: append([]byte("TBHK"), 0xFF, 0, 0, 0),
148+
},
149+
}
150+
for _, tt := range tests {
151+
t.Run(tt.name, func(t *testing.T) {
152+
dir := t.TempDir()
153+
p := filepath.Join(dir, filepath.Base(HintFilePath(dir, tt.fid)))
154+
require.NoError(t, os.WriteFile(p, tt.content, 0o644))
155+
_, err := ReadHintFile(dir, tt.fid)
156+
assert.ErrorIs(t, err, ErrInvalidHintFile)
157+
})
158+
}
159+
}

0 commit comments

Comments
 (0)