Skip to content

Commit 84f20fe

Browse files
kalverrakalverra
authored
[DX-4492] Faster WASM (#22992)
* pre-compile WASM for speed * simplify wasm logic * docker ignore * capabilities/compute tests * Linting, test stability and speed improvements * different wasm binary approach * only commit compressed * delete wasm files * update wasm files * comments * wasm * delete vcs data for wasm builds * fix go.md * fix generate * optimize WASM tests * fix err * fix race
1 parent 827dd86 commit 84f20fe

72 files changed

Lines changed: 880 additions & 461 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ integration/
2222
integration-scripts/
2323
integration-tests/
2424
system-tests/
25+
testdata/
2526

2627
tools/gethnet/datadir/geth
2728
tools/clroot/db.bolt

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ ocr_soak_report.csv
115115
vendor/*
116116

117117
*.wasm
118+
!**/testdata/output.wasm.br
118119
contracts/lcov.info
119120
contracts/out/
120121
/cache

‎core/capabilities/compute/cache.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,9 @@ func newModuleCache(clock clockwork.Clock, tick, timeout time.Duration, evictAft
5757
}
5858

5959
func (mc *moduleCache) start() {
60-
mc.wg.Add(1)
61-
go func() {
62-
defer mc.wg.Done()
60+
mc.wg.Go(func() {
6361
mc.reapLoop()
64-
}()
62+
})
6563
}
6664

6765
func (mc *moduleCache) close() {

‎core/capabilities/compute/cache_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func TestCache(t *testing.T) {
3434
cache.start()
3535
defer cache.close()
3636

37-
binary := wasmtest.CreateTestBinary(t, simpleBinaryCmd, false)
37+
binary := wasmtest.GetTestBinary(t, simpleBinaryCmd, false)
3838
hmod, err := host.NewModule(t.Context(), &host.ModuleConfig{
3939
Logger: logger.Test(t),
4040
IsUncompressed: true,
@@ -45,7 +45,7 @@ func TestCache(t *testing.T) {
4545
mod := &module{
4646
module: hmod,
4747
}
48-
cache.add(id, mod)
48+
require.NoError(t, cache.add(id, mod))
4949

5050
got, ok := cache.get(id)
5151
assert.True(t, ok)
@@ -73,7 +73,7 @@ func TestCache_EvictAfterSize(t *testing.T) {
7373
cache.start()
7474
defer cache.close()
7575

76-
binary := wasmtest.CreateTestBinary(t, simpleBinaryCmd, false)
76+
binary := wasmtest.GetTestBinary(t, simpleBinaryCmd, false)
7777
hmod, err := host.NewModule(t.Context(), &host.ModuleConfig{
7878
Logger: logger.Test(t),
7979
IsUncompressed: true,
@@ -84,7 +84,7 @@ func TestCache_EvictAfterSize(t *testing.T) {
8484
mod := &module{
8585
module: hmod,
8686
}
87-
cache.add(id, mod)
87+
require.NoError(t, cache.add(id, mod))
8888
assert.Len(t, cache.m, 1)
8989

9090
got, ok := cache.get(id)
@@ -116,7 +116,7 @@ func TestCache_AddDuplicatedModule(t *testing.T) {
116116
cache.start()
117117
defer cache.close()
118118

119-
simpleBinary := wasmtest.CreateTestBinary(t, simpleBinaryCmd, false)
119+
simpleBinary := wasmtest.GetTestBinary(t, simpleBinaryCmd, false)
120120
shmod, err := host.NewModule(t.Context(), &host.ModuleConfig{
121121
Logger: logger.Test(t),
122122
IsUncompressed: true,
@@ -136,7 +136,7 @@ func TestCache_AddDuplicatedModule(t *testing.T) {
136136
assert.Equal(t, got, smod)
137137

138138
// Adding a different module but with the same id should not overwrite the existing module
139-
fetchBinary := wasmtest.CreateTestBinary(t, fetchBinaryCmd, false)
139+
fetchBinary := wasmtest.GetTestBinary(t, fetchBinaryCmd, false)
140140
fhmod, err := host.NewModule(t.Context(), &host.ModuleConfig{
141141
Logger: logger.Test(t),
142142
IsUncompressed: true,

‎core/capabilities/compute/compute.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ var (
7575
}, []string{"workflowID", "stepRef"})
7676
)
7777

78-
var _ capabilities.ActionCapability = (*Compute)(nil)
78+
var _ capabilities.ExecutableCapability = (*Compute)(nil)
7979

8080
type FetcherFn func(ctx context.Context, req *host.FetchRequest) (*host.FetchResponse, error)
8181

‎core/capabilities/compute/compute_test.go‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package compute
22

3+
//go:generate go run ../../internal/testutils/wasmtest/generator/main.go -pkg core/capabilities/compute/test/simple/cmd
4+
//go:generate go run ../../internal/testutils/wasmtest/generator/main.go -pkg core/capabilities/compute/test/fetch/cmd
5+
36
import (
47
"context"
58
"encoding/json"
@@ -92,6 +95,7 @@ func setup(t *testing.T, config Config) testHarness {
9295
}
9396

9497
func TestComputeStartAddsToRegistry(t *testing.T) {
98+
t.Parallel()
9599
th := setup(t, defaultConfig)
96100

97101
require.NoError(t, th.compute.Start(t.Context()))
@@ -110,7 +114,7 @@ func TestComputeExecuteMissingConfig(t *testing.T) {
110114
th := setup(t, defaultConfig)
111115
require.NoError(t, th.compute.Start(t.Context()))
112116

113-
binary := wasmtest.CreateTestBinary(t, simpleBinaryCmd, true)
117+
binary := wasmtest.GetTestBinary(t, simpleBinaryCmd, true)
114118

115119
config, err := values.WrapMap(map[string]any{
116120
"binary": binary,
@@ -128,6 +132,7 @@ func TestComputeExecuteMissingConfig(t *testing.T) {
128132
}
129133

130134
func TestComputeExecuteMissingBinary(t *testing.T) {
135+
t.Parallel()
131136
th := setup(t, defaultConfig)
132137

133138
require.NoError(t, th.compute.Start(t.Context()))
@@ -153,7 +158,7 @@ func TestComputeExecute(t *testing.T) {
153158

154159
require.NoError(t, th.compute.Start(t.Context()))
155160

156-
binary := wasmtest.CreateTestBinary(t, simpleBinaryCmd, true)
161+
binary := wasmtest.GetTestBinary(t, simpleBinaryCmd, true)
157162

158163
config, err := values.WrapMap(map[string]any{
159164
"config": []byte(""),
@@ -239,7 +244,7 @@ func TestComputeFetch(t *testing.T) {
239244

240245
require.NoError(t, th.compute.Start(t.Context()))
241246

242-
binary := wasmtest.CreateTestBinary(t, fetchBinaryCmd, true)
247+
binary := wasmtest.GetTestBinary(t, fetchBinaryCmd, true)
243248

244249
config, err := values.WrapMap(map[string]any{
245250
"config": []byte(""),
@@ -312,10 +317,10 @@ func TestCompute_SpendValueRelativeToComputeTime(t *testing.T) {
312317
delay time.Duration
313318
}{
314319
{0},
315-
{time.Second},
316-
{2 * time.Second},
317-
{2_500 * time.Millisecond},
318-
{3 * time.Second},
320+
{50 * time.Millisecond},
321+
{100 * time.Millisecond},
322+
{150 * time.Millisecond},
323+
{200 * time.Millisecond},
319324
}
320325

321326
workflowID := "15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0"
@@ -326,7 +331,7 @@ func TestCompute_SpendValueRelativeToComputeTime(t *testing.T) {
326331
validRequestUUID,
327332
}, "/")
328333
gatewayResp := gatewayResponse(t, msgID, []byte("response body"), privateKey)
329-
binary := wasmtest.CreateTestBinary(t, fetchBinaryCmd, true)
334+
binary := wasmtest.GetTestBinary(t, fetchBinaryCmd, true)
330335

331336
config, err := values.WrapMap(map[string]any{
332337
"config": []byte(""),
@@ -426,7 +431,7 @@ func TestComputeFetchMaxResponseSizeBytes(t *testing.T) {
426431

427432
require.NoError(t, th.compute.Start(t.Context()))
428433

429-
binary := wasmtest.CreateTestBinary(t, fetchBinaryCmd, true)
434+
binary := wasmtest.GetTestBinary(t, fetchBinaryCmd, true)
430435

431436
config, err := values.WrapMap(map[string]any{
432437
"config": []byte(""),

‎core/capabilities/compute/monitoring.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ import (
88

99
"github.com/smartcontractkit/chainlink-common/pkg/beholder"
1010
"github.com/smartcontractkit/chainlink-common/pkg/metrics"
11-
12-
localMonitoring "github.com/smartcontractkit/chainlink/v2/core/monitoring"
1311
)
1412

1513
const timestampKey = "computeTimestamp"
@@ -33,6 +31,6 @@ func (c *computeMetricsLabeler) with(keyValues ...string) *computeMetricsLabeler
3331
}
3432

3533
func (c *computeMetricsLabeler) incrementHTTPRequestCounter(ctx context.Context) {
36-
otelLabels := localMonitoring.KvMapToOtelAttributes(c.Labels)
34+
otelLabels := beholder.OtelAttributes(c.Labels).AsStringAttributes()
3735
c.computeHTTPRequestCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...))
3836
}

‎core/capabilities/compute/test/fetch/cmd/testdata/output.wasm.br‎

6.3 MB
Binary file not shown.

‎core/capabilities/compute/test/simple/cmd/testdata/output.wasm.br‎

6.28 MB
Binary file not shown.

‎core/capabilities/compute/transformer_test.go‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@ import (
1515
)
1616

1717
func Test_NotFoundError(t *testing.T) {
18+
t.Parallel()
1819
nfe := NewNotFoundError("test")
1920
assert.Equal(t, "could not find \"test\" in map", nfe.Error())
2021
}
2122

2223
func Test_popValue(t *testing.T) {
24+
t.Parallel()
2325
m, err := values.NewMap(
2426
map[string]any{
2527
"test": "value",
@@ -28,20 +30,20 @@ func Test_popValue(t *testing.T) {
2830
)
2931
require.NoError(t, err)
3032

31-
t.Run("success", func(t *testing.T) {
33+
t.Run("success", func(t *testing.T) { //nolint:paralleltest // subtests share and mutate map
3234
var gotValue string
3335
gotValue, err = popValue[string](m, "test")
3436
require.NoError(t, err)
3537
assert.Equal(t, "value", gotValue)
3638
})
3739

38-
t.Run("not found", func(t *testing.T) {
40+
t.Run("not found", func(t *testing.T) { //nolint:paralleltest // subtests share and mutate map
3941
_, err = popValue[string](m, "foo")
4042
var nfe *NotFoundError
4143
require.ErrorAs(t, err, &nfe)
4244
})
4345

44-
t.Run("type mismatch", func(t *testing.T) {
46+
t.Run("type mismatch", func(t *testing.T) { //nolint:paralleltest // subtests share and mutate map
4547
_, err = popValue[string](m, "mismatch")
4648
require.Error(t, err)
4749
require.ErrorContains(t, err, "could not unwrap value")
@@ -51,28 +53,29 @@ func Test_popValue(t *testing.T) {
5153
}
5254

5355
func Test_popOptionalValue(t *testing.T) {
56+
t.Parallel()
5457
m, err := values.NewMap(
5558
map[string]any{
5659
"test": "value",
5760
"buzz": "fizz",
5861
},
5962
)
6063
require.NoError(t, err)
61-
t.Run("found value", func(t *testing.T) {
64+
t.Run("found value", func(t *testing.T) { //nolint:paralleltest // subtests share and mutate map
6265
var gotValue string
6366
gotValue, err = popOptionalValue[string](m, "test")
6467
require.NoError(t, err)
6568
assert.Equal(t, "value", gotValue)
6669
})
6770

68-
t.Run("not found returns nil error", func(t *testing.T) {
71+
t.Run("not found returns nil error", func(t *testing.T) { //nolint:paralleltest // subtests share and mutate map
6972
var gotValue string
7073
gotValue, err = popOptionalValue[string](m, "foo")
7174
require.NoError(t, err)
7275
assert.Empty(t, gotValue)
7376
})
7477

75-
t.Run("some other error fails", func(t *testing.T) {
78+
t.Run("some other error fails", func(t *testing.T) { //nolint:paralleltest // subtests share and mutate map
7679
var gotValue int
7780
gotValue, err = popOptionalValue[int](m, "buzz")
7881
require.Error(t, err)
@@ -83,11 +86,13 @@ func Test_popOptionalValue(t *testing.T) {
8386
}
8487

8588
func Test_transformer(t *testing.T) {
89+
t.Parallel()
8690
var (
8791
lgger = logger.Test(t)
8892
emitter = custmsg.NewLabeler()
8993
)
9094
t.Run("success", func(t *testing.T) {
95+
t.Parallel()
9196
giveMap, err := values.NewMap(map[string]any{
9297
"maxMemoryMBs": 1024,
9398
"timeout": "4s",
@@ -126,6 +131,7 @@ func Test_transformer(t *testing.T) {
126131
})
127132

128133
t.Run("success missing optional fields", func(t *testing.T) {
134+
t.Parallel()
129135
giveMap, err := values.NewMap(map[string]any{
130136
"binary": []byte{0x01, 0x02, 0x03},
131137
"config": []byte{0x04, 0x05, 0x06},
@@ -158,6 +164,7 @@ func Test_transformer(t *testing.T) {
158164
})
159165

160166
t.Run("fails parsing timeout", func(t *testing.T) {
167+
t.Parallel()
161168
giveMap, err := values.NewMap(map[string]any{
162169
"timeout": "not a duration",
163170
"binary": []byte{0x01, 0x02, 0x03},
@@ -178,6 +185,7 @@ func Test_transformer(t *testing.T) {
178185
})
179186

180187
t.Run("fails parsing tick interval", func(t *testing.T) {
188+
t.Parallel()
181189
giveMap, err := values.NewMap(map[string]any{
182190
"tickInterval": "not a duration",
183191
"binary": []byte{0x01, 0x02, 0x03},
@@ -198,6 +206,7 @@ func Test_transformer(t *testing.T) {
198206
})
199207

200208
t.Run("invalid tickInterval, applies default", func(t *testing.T) {
209+
t.Parallel()
201210
giveMap, err := values.NewMap(map[string]any{
202211
"tickInterval": "-50ms",
203212
"binary": []byte{0x01, 0x02, 0x03},
@@ -218,6 +227,7 @@ func Test_transformer(t *testing.T) {
218227
})
219228

220229
t.Run("invalid timeout, applies default", func(t *testing.T) {
230+
t.Parallel()
221231
giveMap, err := values.NewMap(map[string]any{
222232
"timeout": "-50ms",
223233
"binary": []byte{0x01, 0x02, 0x03},
@@ -238,6 +248,7 @@ func Test_transformer(t *testing.T) {
238248
})
239249

240250
t.Run("timeout too high, applies default", func(t *testing.T) {
251+
t.Parallel()
241252
giveMap, err := values.NewMap(map[string]any{
242253
"timeout": "1h",
243254
"binary": []byte{0x01, 0x02, 0x03},
@@ -258,6 +269,7 @@ func Test_transformer(t *testing.T) {
258269
})
259270

260271
t.Run("tickInterval too high, applies default", func(t *testing.T) {
272+
t.Parallel()
261273
giveMap, err := values.NewMap(map[string]any{
262274
"tickInterval": "1h",
263275
"binary": []byte{0x01, 0x02, 0x03},
@@ -278,6 +290,7 @@ func Test_transformer(t *testing.T) {
278290
})
279291

280292
t.Run("applies default tick interval if missing", func(t *testing.T) {
293+
t.Parallel()
281294
giveMap, err := values.NewMap(map[string]any{
282295
"binary": []byte{0x01, 0x02, 0x03},
283296
"config": []byte{0x04, 0x05, 0x06},

0 commit comments

Comments
 (0)