-
Notifications
You must be signed in to change notification settings - Fork 237
fix(node): keep node running when a service exits without error #3770
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
534fe6e
242f665
6c2732d
f75d143
c00be34
39f6e6e
fdce2ad
0730600
9494e0d
f3aeadc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
|
rodrodros marked this conversation as resolved.
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| package node_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/NethermindEth/juno/blockchain/networks" | ||
| "github.com/NethermindEth/juno/node" | ||
| "github.com/NethermindEth/juno/utils/log" | ||
| "github.com/sourcegraph/conc" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // fakeService lets each test drive what Run does. | ||
| type fakeService struct { | ||
| run func(ctx context.Context) error | ||
| } | ||
|
|
||
| func (f *fakeService) Run(ctx context.Context) error { return f.run(ctx) } | ||
|
|
||
| // TestStartService covers the three ways a service's Run can return and the | ||
| // expected effect on the node-wide context: | ||
| // - error -> shut the node down | ||
| // - panic -> shut the node down and propagate the panic | ||
| // - nil -> let the service exit without taking the node down | ||
| func TestStartService(t *testing.T) { | ||
| // newNode builds a node through the public constructor with the minimal | ||
| // offline config also used by TestNetworkVerificationOnNonEmptyDB. | ||
| newNode := func(t *testing.T) *node.Node { | ||
| n, err := node.New(&node.Config{ | ||
| DatabasePath: t.TempDir(), | ||
| DBCompression: "zstd", | ||
| Network: networks.Sepolia, | ||
| DisableL1Verification: true, | ||
| SubmittedTransactionsCacheEntryTTL: time.Second, | ||
| }, "test", log.NewLevel(log.INFO)) | ||
| require.NoError(t, err) | ||
| return n | ||
| } | ||
|
|
||
| t.Run("error shuts the node down", func(t *testing.T) { | ||
| n := newNode(t) | ||
| wg := conc.NewWaitGroup() | ||
| ctx, cancel := context.WithCancel(t.Context()) | ||
| defer cancel() | ||
|
|
||
| failing := &fakeService{run: func(context.Context) error { | ||
| return errors.New("boom") | ||
| }} | ||
| n.StartService(wg, ctx, cancel, failing) | ||
| wg.Wait() | ||
|
|
||
| require.Error(t, ctx.Err(), "a service error should shut the node down") | ||
| }) | ||
|
Comment on lines
+23
to
+56
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This tests interact with the services but they don't check anything, just that they node stopped. Please use the zap.Observer utility and then check that the right final error logs where logged, otherwise you've no way of checking node behaviour.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done — now asserting on the emitted logs via
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Small update: with the move to |
||
|
|
||
| t.Run("panic shuts the node down and propagates", func(t *testing.T) { | ||
| n := newNode(t) | ||
| wg := conc.NewWaitGroup() | ||
| ctx, cancel := context.WithCancel(t.Context()) | ||
| defer cancel() | ||
|
|
||
| panicking := &fakeService{run: func(context.Context) error { | ||
| panic("boom") | ||
| }} | ||
| n.StartService(wg, ctx, cancel, panicking) | ||
|
|
||
| require.Panics(t, func() { wg.Wait() }, "a panicking service should propagate the panic") | ||
| require.Error(t, ctx.Err(), "a panicking service should shut the node down") | ||
| }) | ||
|
|
||
| t.Run("clean return keeps the node running", func(t *testing.T) { | ||
| n := newNode(t) | ||
| wg := conc.NewWaitGroup() | ||
| ctx, cancel := context.WithCancel(t.Context()) | ||
| defer cancel() | ||
|
|
||
| // Short-lived service: finishes its work and returns nil. | ||
| exited := make(chan struct{}) | ||
| shortLived := &fakeService{run: func(context.Context) error { | ||
| close(exited) | ||
| return nil | ||
| }} | ||
| // Long-running service: lives until the node is shut down. | ||
| stopped := make(chan struct{}) | ||
| longRunning := &fakeService{run: func(ctx context.Context) error { | ||
| <-ctx.Done() | ||
| close(stopped) | ||
| return nil | ||
| }} | ||
|
|
||
| n.StartService(wg, ctx, cancel, longRunning) | ||
| n.StartService(wg, ctx, cancel, shortLived) | ||
|
|
||
| select { | ||
| case <-exited: // the short-lived service has returned cleanly | ||
| case <-time.After(10 * time.Second): | ||
| t.Fatal("short-lived service did not exit in time") | ||
| } | ||
|
|
||
| // Its clean exit must not take the long-running service down with it. | ||
| select { | ||
| case <-stopped: | ||
| t.Fatal("long-running service stopped: node was shut down unexpectedly") | ||
| default: | ||
| } | ||
| require.NoError(t, ctx.Err(), "a clean exit should not shut the node down") | ||
|
|
||
| // A real shutdown still stops everything. | ||
| cancel() | ||
| wg.Wait() | ||
| select { | ||
| case <-stopped: | ||
| case <-time.After(10 * time.Second): | ||
| t.Fatal("long-running service did not stop in time") | ||
| } | ||
| }) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Agreed. Updated the
service.Servicedoc to drop "Services should be blocking" and spell out the contract instead: a long-running service blocks until ctx is cancelled, a short-lived service may return nil once its work is done without shutting the node down, and an error/panic triggers a node-wide shutdown. Done in 242f665.