Skip to content

Commit fbd6a24

Browse files
authored
fix slow brev shell (#412)
* make brev shell faster and remove unused writeconnectionevent * fix go mod
1 parent 5e6c288 commit fbd6a24

11 files changed

Lines changed: 122 additions & 199 deletions

File tree

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ require (
4444
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7
4545
github.com/wk8/go-ordered-map/v2 v2.0.0
4646
github.com/writeas/go-strip-markdown v2.0.1+incompatible
47-
golang.org/x/crypto v0.52.0
4847
golang.org/x/text v0.37.0
4948
k8s.io/cli-runtime v0.31.1
5049
)
@@ -101,6 +100,7 @@ require (
101100
github.com/x448/float16 v0.8.4 // indirect
102101
github.com/xanzy/ssh-agent v0.3.3 // indirect
103102
golang.org/x/arch v0.8.0 // indirect
103+
golang.org/x/crypto v0.52.0 // indirect
104104
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
105105
golang.org/x/sync v0.20.0 // indirect
106106
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect

pkg/cmd/cmd.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ import (
5757
"github.com/brevdev/brev-cli/pkg/cmd/updatemodel"
5858
"github.com/brevdev/brev-cli/pkg/cmd/upgrade"
5959
"github.com/brevdev/brev-cli/pkg/cmd/version"
60-
"github.com/brevdev/brev-cli/pkg/cmd/writeconnectionevent"
6160
"github.com/brevdev/brev-cli/pkg/config"
6261
"github.com/brevdev/brev-cli/pkg/entity"
6362
"github.com/brevdev/brev-cli/pkg/featureflag"
@@ -339,7 +338,6 @@ func createCmdTree(cmd *cobra.Command, t *terminal.Terminal, loginCmdStore *stor
339338
cmd.AddCommand(healthcheck.NewCmdHealthcheck(t, noLoginCmdStore))
340339

341340
cmd.AddCommand(setupworkspace.NewCmdSetupWorkspace(noLoginCmdStore))
342-
cmd.AddCommand(writeconnectionevent.NewCmdwriteConnectionEvent(t, loginCmdStore))
343341
cmd.AddCommand(updatemodel.NewCmdupdatemodel(t, loginCmdStore))
344342
cmd.AddCommand(feedback.NewCmdFeedback(t, noLoginCmdStore))
345343
}

pkg/cmd/copy/copy.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import (
1818
breverrors "github.com/brevdev/brev-cli/pkg/errors"
1919
"github.com/brevdev/brev-cli/pkg/store"
2020
"github.com/brevdev/brev-cli/pkg/terminal"
21-
"github.com/brevdev/brev-cli/pkg/writeconnectionevent"
2221
"github.com/briandowns/spinner"
2322

2423
"github.com/spf13/cobra"
@@ -99,8 +98,6 @@ func runCopyCommand(t *terminal.Terminal, cstore CopyStore, source, dest string,
9998
return breverrors.WrapAndTrace(err)
10099
}
101100

102-
_ = writeconnectionevent.WriteWCEOnEnv(cstore, workspace.DNS)
103-
104101
err = runSCP(t, sshName, localPath, remotePath, isUpload)
105102
if err != nil {
106103
return breverrors.WrapAndTrace(err)

pkg/cmd/exec/exec.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
breverrors "github.com/brevdev/brev-cli/pkg/errors"
1717
"github.com/brevdev/brev-cli/pkg/store"
1818
"github.com/brevdev/brev-cli/pkg/terminal"
19-
"github.com/brevdev/brev-cli/pkg/writeconnectionevent"
2019
"github.com/hashicorp/go-multierror"
2120

2221
"github.com/spf13/cobra"
@@ -229,7 +228,6 @@ func runExecCommand(t *terminal.Terminal, sstore ExecStore, workspaceNameOrID st
229228
if err != nil {
230229
return breverrors.WrapAndTrace(err)
231230
}
232-
_ = writeconnectionevent.WriteWCEOnEnv(sstore, workspace.DNS)
233231
err = runSSH(sshName, command)
234232
if err != nil {
235233
return breverrors.WrapAndTrace(err)
@@ -263,7 +261,6 @@ func runExecCommand(t *terminal.Terminal, sstore ExecStore, workspaceNameOrID st
263261
"could not connect to instance %q: %w\nPlease check with: brev ls",
264262
workspaceNameOrID, err))
265263
}
266-
_ = writeconnectionevent.WriteWCEOnEnv(sstore, workspace.DNS)
267264
err = runSSH(sshName, command)
268265
if err != nil {
269266
return breverrors.WrapAndTrace(err)

pkg/cmd/open/open.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525
"github.com/brevdev/brev-cli/pkg/store"
2626
"github.com/brevdev/brev-cli/pkg/terminal"
2727
uutil "github.com/brevdev/brev-cli/pkg/util"
28-
"github.com/brevdev/brev-cli/pkg/writeconnectionevent"
2928
"github.com/briandowns/spinner"
3029
"github.com/hashicorp/go-multierror"
3130
"github.com/samber/mo"
@@ -341,10 +340,6 @@ func runOpenCommand(t *terminal.Terminal, tstore OpenStore, wsIDOrName string, s
341340
if err != nil {
342341
return breverrors.WrapAndTrace(err)
343342
}
344-
// we don't care about the error here but should log with sentry
345-
// legacy environments wont support this and cause errrors,
346-
// but we don't want to block the user from using vscode
347-
_ = writeconnectionevent.WriteWCEOnEnv(tstore, string(localIdentifier))
348343
err = openEditorWithSSH(t, string(localIdentifier), projPath, tstore, setupDoneString, editorType)
349344
if err != nil {
350345
if strings.Contains(err.Error(), `"code": executable file not found in $PATH`) {

pkg/cmd/shell/shell.go

Lines changed: 35 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ import (
2020
breverrors "github.com/brevdev/brev-cli/pkg/errors"
2121
"github.com/brevdev/brev-cli/pkg/store"
2222
"github.com/brevdev/brev-cli/pkg/terminal"
23-
"github.com/brevdev/brev-cli/pkg/writeconnectionevent"
24-
2523
"github.com/spf13/cobra"
2624
)
2725

@@ -100,10 +98,26 @@ func runShellCommand(t *terminal.Terminal, sstore ShellStore, workspaceNameOrID
10098
return breverrors.WrapAndTrace(err)
10199
}
102100
}
103-
err = util.PollUntil(s, workspace.ID, "RUNNING", sstore, " waiting for instance to be ready...", pollTimeout)
101+
if workspace.Status != "RUNNING" {
102+
err = util.PollUntil(s, workspace.ID, "RUNNING", sstore, " waiting for instance to be ready...", pollTimeout)
103+
}
104104
if err != nil {
105105
return breverrors.WrapAndTrace(err)
106106
}
107+
108+
localIdentifier := workspace.GetLocalIdentifier()
109+
if host {
110+
localIdentifier = workspace.GetHostIdentifier()
111+
}
112+
sshName := string(localIdentifier)
113+
114+
err = runSSHWithOptions(sshName, host, false)
115+
if err == nil {
116+
trackShellAnalytics(sstore, workspace)
117+
return nil
118+
}
119+
_, _ = fmt.Fprintln(os.Stderr, "\nConnection failed, refreshing SSH config and retrying...")
120+
107121
refreshRes := refresh.RunRefreshAsync(sstore)
108122

109123
workspace, err = util.GetUserWorkspaceByNameOrIDErr(sstore, workspaceNameOrID)
@@ -114,13 +128,6 @@ func runShellCommand(t *terminal.Terminal, sstore ShellStore, workspaceNameOrID
114128
return breverrors.New("Instance is not running")
115129
}
116130

117-
localIdentifier := workspace.GetLocalIdentifier()
118-
if host {
119-
localIdentifier = workspace.GetHostIdentifier()
120-
}
121-
122-
sshName := string(localIdentifier)
123-
124131
err = refreshRes.Await()
125132
if err != nil {
126133
return breverrors.WrapAndTrace(err)
@@ -129,15 +136,16 @@ func runShellCommand(t *terminal.Terminal, sstore ShellStore, workspaceNameOrID
129136
if err != nil {
130137
return breverrors.WrapAndTrace(err)
131138
}
132-
// we don't care about the error here but should log with sentry
133-
// legacy environments wont support this and cause errrors,
134-
// but we don't want to block the user from using the shell
135-
_ = writeconnectionevent.WriteWCEOnEnv(sstore, workspace.DNS)
136139
err = runSSH(sshName, host)
137140
if err != nil {
138141
return breverrors.WrapAndTrace(err)
139142
}
140-
// Call analytics for shell
143+
trackShellAnalytics(sstore, workspace)
144+
145+
return nil
146+
}
147+
148+
func trackShellAnalytics(sstore ShellStore, workspace *entity.Workspace) {
141149
userID := ""
142150
user, err := sstore.GetCurrentUser()
143151
if err != nil {
@@ -146,15 +154,13 @@ func runShellCommand(t *terminal.Terminal, sstore ShellStore, workspaceNameOrID
146154
userID = user.ID
147155
}
148156
data := analytics.EventData{
149-
EventName: "Brev Open",
157+
EventName: "Brev Shell",
150158
UserID: userID,
151159
Properties: map[string]string{
152160
"instanceId": workspace.ID,
153161
},
154162
}
155163
_ = analytics.TrackEvent(data)
156-
157-
return nil
158164
}
159165

160166
func shellIntoExternalNode(t *terminal.Terminal, sstore ShellStore, node *nodev1.ExternalNode) error {
@@ -180,7 +186,7 @@ func shellIntoExternalNode(t *terminal.Terminal, sstore ShellStore, node *nodev1
180186
}
181187

182188
func runSSHWithPort(target string, port int32, identityFile string) error {
183-
sshAgentEval := "eval $(ssh-agent -s)"
189+
sshAgentEval := `if [ -z "$SSH_AUTH_SOCK" ]; then eval $(ssh-agent -s) > /dev/null; fi`
184190
cmd := fmt.Sprintf("%s && ssh -i %q -o StrictHostKeyChecking=no -p %d %s", sshAgentEval, identityFile, port, target)
185191

186192
sshCmd := exec.Command("bash", "-c", cmd) //nolint:gosec //cmd is constructed from API data
@@ -201,13 +207,17 @@ func runSSHWithPort(target string, port int32, identityFile string) error {
201207
}
202208

203209
func runSSH(sshAlias string, host bool) error {
204-
sshAgentEval := "eval $(ssh-agent -s)"
210+
return runSSHWithOptions(sshAlias, host, true)
211+
}
212+
213+
func runSSHWithOptions(sshAlias string, host bool, printFailureAdvice bool) error {
214+
sshAgentEval := `if [ -z "$SSH_AUTH_SOCK" ]; then eval $(ssh-agent -s) > /dev/null; fi`
205215
var cmd string
206216
if host {
207-
cmd = fmt.Sprintf("%s && ssh %s", sshAgentEval, sshAlias)
217+
cmd = fmt.Sprintf("%s && ssh -o ConnectTimeout=5 %s", sshAgentEval, sshAlias)
208218
} else {
209219
// SSH into VM and respect container WORKDIR if containerized, otherwise use default directory
210-
cmd = fmt.Sprintf("%s && ssh -t %s 'DIR=$(readlink -f /proc/1/cwd 2>/dev/null || pwd); cd \"$DIR\" || echo \"Warning: Could not access container directory\" >&2; exec -l ${SHELL:-/bin/sh}'", sshAgentEval, sshAlias)
220+
cmd = fmt.Sprintf("%s && ssh -t -o ConnectTimeout=5 %s 'DIR=$(readlink -f /proc/1/cwd 2>/dev/null || pwd); cd \"$DIR\" || echo \"Warning: Could not access container directory\" >&2; exec -l ${SHELL:-/bin/sh}'", sshAgentEval, sshAlias)
211221
}
212222

213223
var stderrBuf bytes.Buffer
@@ -223,6 +233,9 @@ func runSSH(sshAlias string, host bool) error {
223233

224234
err = sshCmd.Run()
225235
if err != nil {
236+
if !printFailureAdvice {
237+
return breverrors.WrapAndTrace(err)
238+
}
226239
stderrStr := stderrBuf.String()
227240
if strings.Contains(stderrStr, "unix_listener") || strings.Contains(stderrStr, "path too long") {
228241
fmt.Fprintf(os.Stderr, "\nbrev shell failed: SSH ControlPath socket path is too long for this system.\n")

pkg/cmd/util/ssh.go

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package util
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
67
"os/exec"
@@ -14,6 +15,14 @@ import (
1415
"github.com/briandowns/spinner"
1516
)
1617

18+
var (
19+
sshAvailabilityConnectTimeoutSeconds = 3
20+
sshAvailabilityAttemptTimeout = 5 * time.Second
21+
sshAvailabilityWaitDelay = time.Second
22+
sshAvailabilityRetrySleep = time.Second
23+
sshAvailabilityMaxAttempts = 20
24+
)
25+
1726
// WorkspacePollingStore is the minimal interface needed for polling workspace state
1827
type WorkspacePollingStore interface {
1928
GetWorkspace(workspaceID string) (*entity.Workspace, error)
@@ -56,27 +65,45 @@ func WaitForSSHToBeAvailable(sshAlias string, s *spinner.Spinner) error {
5665
s.Suffix = " waiting for SSH connection to be available"
5766
s.Start()
5867
for {
59-
cmd := exec.Command("ssh", "-o", "ConnectTimeout=10", sshAlias, "echo", " ")
68+
attempt := counter + 1
69+
ctx, cancel := context.WithTimeout(context.Background(), sshAvailabilityAttemptTimeout)
70+
cmd := exec.CommandContext(ctx, "ssh",
71+
"-T",
72+
"-o", fmt.Sprintf("ConnectTimeout=%d", sshAvailabilityConnectTimeoutSeconds),
73+
"-o", "ConnectionAttempts=1",
74+
"-o", "BatchMode=yes",
75+
"-o", "NumberOfPasswordPrompts=0",
76+
"-o", "RequestTTY=no",
77+
"-o", "LogLevel=ERROR",
78+
sshAlias,
79+
"true",
80+
)
81+
cmd.WaitDelay = sshAvailabilityWaitDelay
6082
out, err := cmd.CombinedOutput()
83+
timedOut := ctx.Err() == context.DeadlineExceeded
84+
cancel()
6185
if err == nil {
6286
s.Stop()
6387
return nil
6488
}
6589

66-
outputStr := string(out)
67-
lines := strings.Split(outputStr, "\n")
68-
stdErr := outputStr
69-
if len(lines) > 1 {
70-
stdErr = lines[1]
90+
stdErr := strings.TrimSpace(string(out))
91+
if timedOut {
92+
stdErr = fmt.Sprintf("SSH attempt %d timed out after %s", attempt, sshAvailabilityAttemptTimeout)
93+
} else if stdErr == "" {
94+
stdErr = err.Error()
7195
}
7296

73-
if counter == 40 || !store.SatisfactorySSHErrMessage(stdErr) {
97+
if counter == sshAvailabilityMaxAttempts || (!timedOut && !store.SatisfactorySSHErrMessage(stdErr)) {
7498
s.Stop()
7599
return breverrors.WrapAndTrace(errors.New("\n" + stdErr))
76100
}
77101

102+
s.Stop()
103+
_, _ = fmt.Fprintf(s.Writer, "still waiting for SSH connection (attempt %d failed; retrying)\n", attempt)
78104
counter++
79-
time.Sleep(1 * time.Second)
105+
time.Sleep(sshAvailabilityRetrySleep)
106+
s.Start()
80107
}
81108
}
82109

pkg/cmd/util/ssh_test.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package util
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"strings"
7+
"testing"
8+
"time"
9+
10+
"github.com/briandowns/spinner"
11+
)
12+
13+
func TestWaitForSSHToBeAvailableTimesOutStuckSSHAttempt(t *testing.T) {
14+
dir := t.TempDir()
15+
fakeSSH := filepath.Join(dir, "ssh")
16+
err := os.WriteFile(fakeSSH, []byte("#!/bin/sh\nexec sleep 5\n"), 0o755)
17+
if err != nil {
18+
t.Fatalf("write fake ssh: %v", err)
19+
}
20+
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
21+
22+
originalAttemptTimeout := sshAvailabilityAttemptTimeout
23+
originalWaitDelay := sshAvailabilityWaitDelay
24+
originalRetrySleep := sshAvailabilityRetrySleep
25+
originalMaxAttempts := sshAvailabilityMaxAttempts
26+
sshAvailabilityAttemptTimeout = 50 * time.Millisecond
27+
sshAvailabilityWaitDelay = 50 * time.Millisecond
28+
sshAvailabilityRetrySleep = 0
29+
sshAvailabilityMaxAttempts = 0
30+
t.Cleanup(func() {
31+
sshAvailabilityAttemptTimeout = originalAttemptTimeout
32+
sshAvailabilityWaitDelay = originalWaitDelay
33+
sshAvailabilityRetrySleep = originalRetrySleep
34+
sshAvailabilityMaxAttempts = originalMaxAttempts
35+
})
36+
37+
s := spinner.New(spinner.CharSets[9], 100*time.Millisecond)
38+
start := time.Now()
39+
err = WaitForSSHToBeAvailable("slow-host", s)
40+
elapsed := time.Since(start)
41+
42+
if err == nil {
43+
t.Fatal("expected stuck ssh attempt to fail")
44+
}
45+
if elapsed > 500*time.Millisecond {
46+
t.Fatalf("expected stuck ssh attempt to be killed quickly, took %v", elapsed)
47+
}
48+
if !strings.Contains(err.Error(), "timed out after") {
49+
t.Fatalf("expected timeout error, got %v", err)
50+
}
51+
}

pkg/cmd/writeconnectionevent/writeconnectionevent.go

Lines changed: 0 additions & 44 deletions
This file was deleted.

0 commit comments

Comments
 (0)