Skip to content

Commit 4ee2346

Browse files
marcushillmarcushill
authored
[commitlog] Add the ability to configure pools before allocation (#4072)
* Add the ability to configure pools before allocation There are several places where we allocate large pools of memory in constructors and provide setters to override those pools. In general, this is fine as it allows us to provide larger or smaller pools. However, in super memory constrained settings, we prefer to avoid the initial alloc and GC all together and provide the properly sized pools to the constructor. This PR provides the ability to do this for certain properties of fs.Options and commitlog.Options.
1 parent ac4617d commit 4ee2346

2 files changed

Lines changed: 125 additions & 19 deletions

File tree

‎src/dbnode/persist/fs/commitlog/options.go‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -94,27 +94,72 @@ type options struct {
9494
failureCallback FailureCallback
9595
}
9696

97+
type optionsInput struct {
98+
fsOptions fs.Options
99+
// allows differentiating between explicitly nil and unset
100+
fsOptionsSet bool
101+
102+
identPoolOpts ident.PoolOptions
103+
bytePoolOptions pool.ObjectPoolOptions
104+
}
105+
106+
// OptionSetter is a function that modifies the behavior of NewOptions
107+
type OptionSetter func(o *optionsInput)
108+
109+
// WithFileSystemOptions is an OptionsSetter that provides custom fs.Options
110+
// Passing nil will be equivalent to calling Options.SetFilesystemOptions(nil)
111+
func WithFileSystemOptions(o fs.Options) OptionSetter {
112+
return func(input *optionsInput) {
113+
input.fsOptions = o
114+
input.fsOptionsSet = true
115+
}
116+
}
117+
118+
// WithIdentPoolOptions is an OptionsSetter that provides options to the IdentifierPool
119+
func WithIdentPoolOptions(o ident.PoolOptions) OptionSetter {
120+
return func(input *optionsInput) {
121+
input.identPoolOpts = o
122+
}
123+
}
124+
125+
// WithBytesPoolOptions is an OptionsSetter that provides options to BytesPool
126+
func WithBytesPoolOptions(o pool.ObjectPoolOptions) OptionSetter {
127+
return func(input *optionsInput) {
128+
input.bytePoolOptions = o
129+
}
130+
}
131+
97132
// NewOptions creates new commit log options
98-
func NewOptions() Options {
133+
func NewOptions(setters ...OptionSetter) Options {
134+
presetOptions := optionsInput{}
135+
for _, setter := range setters {
136+
setter(&presetOptions)
137+
}
138+
139+
if !presetOptions.fsOptionsSet && presetOptions.fsOptions == nil {
140+
presetOptions.fsOptions = fs.NewOptions()
141+
}
142+
99143
o := &options{
100144
clockOpts: clock.NewOptions(),
101145
instrumentOpts: instrument.NewOptions(),
102146
blockSize: defaultBlockSize,
103-
fsOpts: fs.NewOptions(),
147+
fsOpts: presetOptions.fsOptions,
104148
strategy: defaultStrategy,
105149
failureMode: defaultFailureStrategy,
106150
flushSize: defaultFlushSize,
107151
flushInterval: defaultFlushInterval,
108152
backlogQueueSize: defaultBacklogQueueSize,
109153
backlogQueueChannelSize: defaultBacklogQueueChannelSize,
110-
bytesPool: pool.NewCheckedBytesPool(nil, nil, func(s []pool.Bucket) pool.BytesPool {
111-
return pool.NewBytesPool(s, nil)
154+
bytesPool: pool.NewCheckedBytesPool(nil, presetOptions.bytePoolOptions, func(s []pool.Bucket) pool.BytesPool {
155+
return pool.NewBytesPool(s, presetOptions.bytePoolOptions)
112156
}),
113157
readConcurrency: defaultReadConcurrency,
114158
failureCallback: nil,
115159
}
160+
116161
o.bytesPool.Init()
117-
o.identPool = ident.NewPool(o.bytesPool, ident.PoolOptions{})
162+
o.identPool = ident.NewPool(o.bytesPool, presetOptions.identPoolOpts)
118163
return o
119164
}
120165

‎src/dbnode/persist/fs/options.go‎

Lines changed: 75 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ const (
7676

7777
var (
7878
defaultFilePathPrefix = os.TempDir()
79-
defaultNewFileMode = os.FileMode(0666)
80-
defaultNewDirectoryMode = os.ModeDir | os.FileMode(0755)
79+
defaultNewFileMode = os.FileMode(0o666)
80+
defaultNewDirectoryMode = os.ModeDir | os.FileMode(0o755)
8181
defaultFSTWriterOptions = fst.WriterOptions{}
8282

8383
errTagEncoderPoolNotSet = errors.New("tag encoder pool is not set")
@@ -111,16 +111,77 @@ type options struct {
111111
encodingOptions msgpack.LegacyEncodingOptions
112112
}
113113

114+
type optionsInput struct {
115+
tagEncoderPool serialize.TagEncoderPool
116+
tagDecoderPool serialize.TagDecoderPool
117+
fstOptions fst.Options
118+
119+
// the bools allow explicitly setting the field to nil
120+
tagEncoderPoolSet bool
121+
tagDecoderPoolSet bool
122+
fstOptionsSet bool
123+
}
124+
125+
// OptionSetter is a function that modifies the behavior of NewOptions
126+
type OptionSetter func(o *optionsInput)
127+
128+
// WithTagEncoderPool is an OptionSetter that provides custom serialize.TagEncoderPool
129+
// Passing nil will be equivalent to calling Options.SetTagEncoderPool(nil)
130+
// on the result of NewOptions
131+
func WithTagEncoderPool(o serialize.TagEncoderPool) OptionSetter {
132+
return func(input *optionsInput) {
133+
input.tagEncoderPool = o
134+
input.tagEncoderPoolSet = true
135+
}
136+
}
137+
138+
// WithTagDecodePool is an OptionSetter that provides custom serialize.TagDecoder
139+
// Passing nil will be equivalent to calling Options.SetTagDecoderPool(nil)
140+
// on the result of NewOptions
141+
func WithTagDecodePool(o serialize.TagDecoderPool) OptionSetter {
142+
return func(input *optionsInput) {
143+
input.tagDecoderPool = o
144+
input.tagDecoderPoolSet = true
145+
}
146+
}
147+
148+
// WithFstOptions is an OptionSetter that provides custom fst.Options
149+
// Passing nil will be equivalent to calling Options.SetFSTOptions(nil)
150+
// on the result of NewOptions
151+
func WithFstOptions(o fst.Options) OptionSetter {
152+
return func(input *optionsInput) {
153+
input.fstOptions = o
154+
input.fstOptionsSet = true
155+
}
156+
}
157+
114158
// NewOptions creates a new set of fs options
115-
func NewOptions() Options {
116-
tagEncoderPool := serialize.NewTagEncoderPool(
117-
serialize.NewTagEncoderOptions(), pool.NewObjectPoolOptions())
118-
tagEncoderPool.Init()
119-
tagDecoderPool := serialize.NewTagDecoderPool(
120-
serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}),
121-
pool.NewObjectPoolOptions())
122-
tagDecoderPool.Init()
123-
fstOptions := fst.NewOptions()
159+
func NewOptions(setters ...OptionSetter) Options {
160+
input := optionsInput{}
161+
for _, setter := range setters {
162+
setter(&input)
163+
}
164+
if !input.tagEncoderPoolSet && input.tagEncoderPool == nil {
165+
input.tagEncoderPool = serialize.NewTagEncoderPool(
166+
serialize.NewTagEncoderOptions(), pool.NewObjectPoolOptions())
167+
}
168+
169+
if !input.tagDecoderPoolSet && input.tagDecoderPool == nil {
170+
input.tagDecoderPool = serialize.NewTagDecoderPool(
171+
serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}),
172+
pool.NewObjectPoolOptions())
173+
}
174+
175+
if !input.fstOptionsSet && input.fstOptions == nil {
176+
input.fstOptions = fst.NewOptions()
177+
}
178+
179+
if input.tagEncoderPool != nil {
180+
input.tagEncoderPool.Init()
181+
}
182+
if input.tagDecoderPool != nil {
183+
input.tagDecoderPool.Init()
184+
}
124185

125186
return &options{
126187
clockOpts: clock.NewOptions(),
@@ -140,9 +201,9 @@ func NewOptions() Options {
140201
seekReaderBufferSize: defaultSeekReaderBufferSize,
141202
mmapEnableHugePages: defaultMmapEnableHugePages,
142203
mmapHugePagesThreshold: defaultMmapHugePagesThreshold,
143-
tagEncoderPool: tagEncoderPool,
144-
tagDecoderPool: tagDecoderPool,
145-
fstOptions: fstOptions,
204+
tagEncoderPool: input.tagEncoderPool,
205+
tagDecoderPool: input.tagDecoderPool,
206+
fstOptions: input.fstOptions,
146207
fstWriterOptions: defaultFSTWriterOptions,
147208
indexReaderAutovalidateIndexSegments: defaultIndexReaderAutovalidateIndexSegments,
148209
encodingOptions: msgpack.DefaultLegacyEncodingOptions,

0 commit comments

Comments
 (0)