update statediff algorithm for wavesrv / remote instances (#530)

* remote statemap from waveshell server (diff against initial state)

* move ShellStatePtr from sstore to packet so it can be passed over the wire

* add finalstatebaseptr to cmddone

* much improved diff computation code on wavesrv side

* fix displayname -- now using hash

* add comments, change a couple msh.WriteToPtyBuffer calls to log.Printfs

Author: sawka

waveshell/pkg/packet/packet.go

@@ -743,13 +743,14 @@ func MakeCmdFinalPacket(ck base.CommandKey) *CmdFinalPacketType {
 }
 
 type CmdDonePacketType struct {
-	Type           string          `json:"type"`
-	Ts             int64           `json:"ts"`
-	CK             base.CommandKey `json:"ck"`
-	ExitCode       int             `json:"exitcode"`
-	DurationMs     int64           `json:"durationms"`
-	FinalState     *ShellState     `json:"finalstate,omitempty"`
-	FinalStateDiff *ShellStateDiff `json:"finalstatediff,omitempty"`
+	Type              string          `json:"type"`
+	Ts                int64           `json:"ts"`
+	CK                base.CommandKey `json:"ck"`
+	ExitCode          int             `json:"exitcode"`
+	DurationMs        int64           `json:"durationms"`
+	FinalState        *ShellState     `json:"finalstate,omitempty"`
+	FinalStateDiff    *ShellStateDiff `json:"finalstatediff,omitempty"`
+	FinalStateBasePtr *ShellStatePtr  `json:"finalstatebaseptr,omitempty"`
 }
 
 func (*CmdDonePacketType) GetType() string {
@@ -814,10 +815,10 @@ type RunPacketType struct {
 	Type          string          `json:"type"`
 	ReqId         string          `json:"reqid"`
 	CK            base.CommandKey `json:"ck"`
-	ShellType     string          `json:"shelltype"` // new in v0.6.0 (either "bash" or "zsh") (set by remote.go)
+	ShellType     string          `json:"shelltype"` // added in Wave v0.6.0 (either "bash" or "zsh") (set by remote.go)
 	Command       string          `json:"command"`
 	State         *ShellState     `json:"state,omitempty"`
-	StateDiff     *ShellStateDiff `json:"statediff,omitempty"`
+	StatePtr      *ShellStatePtr  `json:"stateptr,omitempty"`      // added in Wave v0.7.2
 	StateComplete bool            `json:"statecomplete,omitempty"` // set to true if state is complete (the default env should not be set)
 	UsePty        bool            `json:"usepty,omitempty"`
 	TermOpts      *TermOpts       `json:"termopts,omitempty"`

waveshell/pkg/packet/shellstate.go

@@ -40,6 +40,18 @@ type ShellState struct {
 	HashVal   string `json:"-"`
 }
 
+type ShellStatePtr struct {
+	BaseHash    string   `json:"basehash"`
+	DiffHashArr []string `json:"diffhasharr,omitempty"`
+}
+
+func (ssptr *ShellStatePtr) IsEmpty() bool {
+	if ssptr == nil || ssptr.BaseHash == "" {
+		return true
+	}
+	return false
+}
+
 func (state ShellState) ApproximateSize() int64 {
 	return int64(len(state.Version) + len(state.Cwd) + len(state.ShellVars) + len(state.Aliases) + len(state.Funcs) + len(state.Error))
 }

waveshell/pkg/server/server.go

@@ -50,7 +50,6 @@ type MServer struct {
 	Sender              *packet.PacketSender
 	ClientMap           map[base.CommandKey]*shexec.ClientProc
 	Debug               bool
-	StateMap            *ShellStateMap
 	WriteErrorCh        chan bool // closed if there is a I/O write error
 	WriteErrorChOnce    *sync.Once
 	Done                bool
@@ -338,11 +337,6 @@ func (m *MServer) reinit(reqId string, shellType string) {
 		m.Sender.SendErrorResponse(reqId, fmt.Errorf("error initializing shell: %w", err))
 		return
 	}
-	err = m.StateMap.SetCurrentState(ssPk.State.GetShellType(), ssPk.State)
-	if err != nil {
-		m.Sender.SendErrorResponse(reqId, fmt.Errorf("error setting current state: %w", err))
-		return
-	}
 	ssPk.RespId = reqId
 	m.Sender.SendPacket(ssPk)
 }
@@ -710,33 +704,30 @@ func (m *MServer) ProcessRpcPacket(pk packet.RpcPacketType) {
 	m.Sender.SendErrorResponse(reqId, fmt.Errorf("invalid rpc type '%s'", pk.GetType()))
 }
 
-func (m *MServer) clientPacketCallback(shellType string, pk packet.PacketType) {
+func (m *MServer) clientPacketCallback(shellType string, pk packet.PacketType, runPk *packet.RunPacketType) {
 	if pk.GetType() != packet.CmdDonePacketStr {
 		return
 	}
 	donePk := pk.(*packet.CmdDonePacketType)
 	if donePk.FinalState == nil {
 		return
 	}
-	stateHash, curState := m.StateMap.GetCurrentState(shellType)
-	if curState == nil {
+	initialState := runPk.State
+	if initialState == nil {
 		return
 	}
-	sapi, err := shellapi.MakeShellApi(curState.GetShellType())
+	initialStateHash := initialState.GetHashVal(false)
+	sapi, err := shellapi.MakeShellApi(initialState.GetShellType())
 	if err != nil {
 		return
 	}
-	diff, err := sapi.MakeShellStateDiff(curState, stateHash, donePk.FinalState)
+	diff, err := sapi.MakeShellStateDiff(initialState, initialStateHash, donePk.FinalState)
 	if err != nil {
 		return
 	}
 	donePk.FinalState = nil
 	donePk.FinalStateDiff = diff
-}
-
-func (m *MServer) isShellInitialized(shellType string) bool {
-	_, curState := m.StateMap.GetCurrentState(shellType)
-	return curState != nil
+	donePk.FinalStateBasePtr = runPk.StatePtr
 }
 
 func (m *MServer) runCommand(runPacket *packet.RunPacketType) {
@@ -786,7 +777,7 @@ func (m *MServer) runCommand(runPacket *packet.RunPacketType) {
 		}()
 		shexec.SendRunPacketAndRunData(context.Background(), cproc.Input, runPacket)
 		cproc.ProxySingleOutput(runPacket.CK, m.Sender, func(pk packet.PacketType) {
-			m.clientPacketCallback(runPacket.ShellType, pk)
+			m.clientPacketCallback(runPacket.ShellType, pk, runPacket)
 		})
 	}()
 }
@@ -849,7 +840,6 @@ func RunServer() (int, error) {
 	server := &MServer{
 		Lock:                &sync.Mutex{},
 		ClientMap:           make(map[base.CommandKey]*shexec.ClientProc),
-		StateMap:            MakeShellStateMap(),
 		Debug:               debug,
 		WriteErrorCh:        make(chan bool),
 		WriteErrorChOnce:    &sync.Once{},

waveshell/pkg/shellapi/shellapi.go

@@ -63,6 +63,9 @@ type ShellStateOutput struct {
 	Error      string
 }
 
+// some timing info
+// MakeShellStateDiff takes ~1ms to run (even on a large diff)
+
 type ShellApi interface {
 	GetShellType() string
 	MakeExitTrap(fdNum int) (string, []byte)

waveshell/pkg/statediff/linediff.go

@@ -186,6 +186,9 @@ func readEncodedStringArray(buf *bytes.Buffer) ([]string, error) {
 
 func (rtn *LineDiffType) Decode(diffBytes []byte) error {
 	rtn.Clear()
+	if len(diffBytes) == 0 {
+		return nil
+	}
 	r := bytes.NewBuffer(diffBytes)
 	version, err := binary.ReadUvarint(r)
 	if err != nil {

waveshell/pkg/statediff/mapdiff.go

@@ -29,7 +29,7 @@ func (diff *MapDiffType) Clear() {
 }
 
 func (diff MapDiffType) Dump() {
-	fmt.Printf("VAR-DIFF\n")
+	fmt.Printf("VAR-DIFF +%d -%d\n", len(diff.ToAdd), len(diff.ToRemove))
 	for name, val := range diff.ToAdd {
 		fmt.Printf("  add[%s] %s\n", name, val)
 	}
@@ -111,6 +111,9 @@ func (diff MapDiffType) Encode() []byte {
 
 func (diff *MapDiffType) Decode(diffBytes []byte) error {
 	diff.Clear()
+	if len(diffBytes) == 0 {
+		return nil
+	}
 	r := bytes.NewBuffer(diffBytes)
 	version, err := binpack.UnpackUInt(r)
 	if err != nil {

wavesrv/pkg/cmdrunner/cmdrunner.go

@@ -3862,7 +3862,7 @@ func doAsyncResetCommand(msh *remote.MShellProc, opts connectOptsType, cmd *ssto
 		rtnErr = err
 		return
 	}
-	newStatePtr := sstore.ShellStatePtr{
+	newStatePtr := packet.ShellStatePtr{
 		BaseHash: ssPk.State.GetHashVal(false),
 	}
 	if opts.Verbose && origStatePtr != nil {

wavesrv/pkg/cmdrunner/resolver.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/waveshell/pkg/packet"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/remote"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/sstore"
@@ -42,7 +43,7 @@ type ResolvedRemote struct {
 	RState      remote.RemoteRuntimeState
 	RemoteCopy  *sstore.RemoteType
 	ShellType   string // default remote shell preference
-	StatePtr    *sstore.ShellStatePtr
+	StatePtr    *packet.ShellStatePtr
 	FeState     map[string]string
 }
 
@@ -491,7 +492,7 @@ func ResolveRemoteFromPtr(ctx context.Context, rptr *sstore.RemotePtrType, sessi
 				rtn.StatePtr = nil
 				rtn.FeState = nil
 			} else {
-				rtn.StatePtr = &sstore.ShellStatePtr{BaseHash: ri.StateBaseHash, DiffHashArr: ri.StateDiffHashArr}
+				rtn.StatePtr = &packet.ShellStatePtr{BaseHash: ri.StateBaseHash, DiffHashArr: ri.StateDiffHashArr}
 				rtn.FeState = ri.FeState
 				rtn.ShellType = ri.ShellType
 			}