Skip to content

Commit 0c3231e

Browse files
authored
Add remote log stream support (#5)
1 parent 2008a7b commit 0c3231e

23 files changed

Lines changed: 1326 additions & 31 deletions

File tree

api/api.go

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ import (
55
)
66

77
type SetupRequest struct {
8-
Envs map[string]string `json:"envs,omitempty"`
9-
Network spec.Network `json:"network"`
10-
Platform spec.Platform `json:"platform,omitempty"`
11-
Volumes []*spec.Volume `json:"volumes,omitempty"`
12-
Secrets []string `json:"secrets,omitempty"`
8+
Envs map[string]string `json:"envs,omitempty"`
9+
Network spec.Network `json:"network"`
10+
Platform spec.Platform `json:"platform,omitempty"`
11+
Volumes []*spec.Volume `json:"volumes,omitempty"`
12+
Secrets []string `json:"secrets,omitempty"`
13+
LogConfig LogConfig `json:"log_config,omitempty"`
14+
TIConfig TIConfig `json:"ti_config,omitempty"`
1315
}
1416

1517
type SetupResponse struct{}
@@ -29,6 +31,10 @@ type StartStepRequest struct {
2931
Run RunConfig `json:"run,omitempty"`
3032
RunTest RunTestConfig `json:"run_test,omitempty"`
3133

34+
LogKey string `json:"log_key,omitempty"`
35+
OutputVars []string `json:"output_var,omitempty"`
36+
Timeout int `json:"timeout,omitempty"` // step timeout in seconds
37+
3238
// Valid only for steps running on docker container
3339
Auth *spec.Auth `json:"auth,omitempty"`
3440
CPUPeriod int64 `json:"cpu_period,omitempty"`
@@ -82,3 +88,15 @@ type RunTestConfig struct {
8288
RunOnlySelectedTests bool `json:"run_only_selected_tests,omitempty"`
8389
TestAnnotations string `json:"test_annotations,omitempty"`
8490
}
91+
92+
type LogConfig struct {
93+
AccountID string `json:"account_id,omitempty"`
94+
IndirectUpload bool `json:"indirect_upload,omitempty"` // Whether to directly upload via signed link or using log service
95+
URL string `json:"url,omitempty"`
96+
Token string `json:"token,omitempty"`
97+
}
98+
99+
type TIConfig struct {
100+
URL string `json:"url,omitempty"`
101+
Token string `json:"token,omitempty"`
102+
}

cli/certs/certs.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ func GenerateCert(host string, ca *Certificate) (*Certificate, error) {
6868
logrus.
6969
WithError(certOutErr).
7070
Errorln("cannot pem encode certOut")
71+
return nil, certOutErr
7172
}
7273

7374
keyOut := new(bytes.Buffer)
@@ -79,6 +80,7 @@ func GenerateCert(host string, ca *Certificate) (*Certificate, error) {
7980
logrus.
8081
WithError(keyOutErr).
8182
Errorln("cannot pem encode keyout")
83+
return nil, keyOutErr
8284
}
8385

8486
return &Certificate{
@@ -119,6 +121,7 @@ func GenerateCA() (*Certificate, error) {
119121
logrus.
120122
WithError(certOutErr).
121123
Errorln("cannot pem encode certOut")
124+
return nil, certOutErr
122125
}
123126
keyOut := new(bytes.Buffer)
124127
keyOutErr := pem.Encode(keyOut, &pem.Block{
@@ -129,6 +132,7 @@ func GenerateCA() (*Certificate, error) {
129132
logrus.
130133
WithError(keyOutErr).
131134
Errorln("cannot pem encode keyout")
135+
return nil, keyOutErr
132136
}
133137
return &Certificate{
134138
Cert: certOut.Bytes(),

cli/client/client.go

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ import (
1515
)
1616

1717
type clientCommand struct {
18-
envfile string
19-
runStage bool
18+
envfile string
19+
runStage bool
20+
remoteLog bool
2021
}
2122

2223
func (c *clientCommand) run(*kingpin.ParseContext) error {
@@ -44,7 +45,7 @@ func (c *clientCommand) run(*kingpin.ParseContext) error {
4445
}
4546

4647
if c.runStage {
47-
return runStage(client)
48+
return runStage(client, c.remoteLog)
4849
}
4950
return checkServerHealth(client)
5051
}
@@ -53,7 +54,7 @@ func checkServerHealth(client *HTTPClient) error {
5354
return client.Health(context.Background())
5455
}
5556

56-
func runStage(client *HTTPClient) error {
57+
func runStage(client *HTTPClient, remoteLog bool) error { // nolint:funlen
5758
ctx := context.Background()
5859
defer func() {
5960
logrus.Infof("Starting destroy")
@@ -77,6 +78,14 @@ func runStage(client *HTTPClient) error {
7778
ID: "drone",
7879
},
7980
}
81+
if remoteLog {
82+
setupParams.LogConfig = api.LogConfig{
83+
URL: "http://localhost:8079",
84+
AccountID: "kmpy",
85+
Token: "token",
86+
IndirectUpload: true,
87+
}
88+
}
8089
logrus.Infof("Starting setup")
8190
if _, err := client.Setup(ctx, setupParams); err != nil {
8291
logrus.WithError(err).Errorln("setup call failed")
@@ -97,6 +106,7 @@ func runStage(client *HTTPClient) error {
97106
},
98107
},
99108
WorkingDir: "/drone/src",
109+
LogKey: sid1,
100110
}
101111
s1.Run.Command = []string{"set -xe; pwd; echo drone; echo hello world > foo; cat foo"}
102112
s1.Run.Entrypoint = []string{"sh", "-c"}
@@ -128,6 +138,7 @@ func runStage(client *HTTPClient) error {
128138
},
129139
},
130140
WorkingDir: "/drone/src",
141+
LogKey: sid2,
131142
}
132143
s2.Run.Command = []string{"set -xe; pwd; cat foo"}
133144
s2.Run.Entrypoint = []string{"sh", "-c"}
@@ -159,4 +170,6 @@ func Register(app *kingpin.Application) {
159170
StringVar(&c.envfile)
160171
cmd.Flag("stage", "Run a stage").
161172
BoolVar(&c.runStage)
173+
cmd.Flag("remotelog", "Enable remote logging if client runs in stage mode").
174+
BoolVar(&c.remoteLog)
162175
}

engine/exec/exec.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,6 @@ func Run(ctx context.Context, step *spec.Step, output io.Writer) (*runtime.State
2828
return nil, err
2929
}
3030

31-
if step.Detach {
32-
return &runtime.State{Exited: false}, nil
33-
}
34-
3531
err := cmd.Wait()
3632
if err == nil {
3733
return &runtime.State{ExitCode: 0, Exited: true}, nil

executor/common.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package executor
2+
3+
import (
4+
"errors"
5+
6+
"github.com/harness/lite-engine/livelog"
7+
)
8+
9+
func getNudges() []livelog.Nudge {
10+
// <search-term> <resolution> <error-msg>
11+
return []livelog.Nudge{
12+
livelog.NewNudge("[Kk]illed", "Increase memory resources for the step", errors.New("out of memory")),
13+
livelog.NewNudge(".*git.* SSL certificate problem",
14+
"Set sslVerify to false in CI codebase properties", errors.New("SSL certificate error")),
15+
livelog.NewNudge("Cannot connect to the Docker daemon",
16+
"Setup dind if it's not running. If dind is running, privileged should be set to true",
17+
errors.New("could not connect to the docker daemon")),
18+
}
19+
}

executor/run.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,19 @@ package executor
22

33
import (
44
"context"
5-
"os"
5+
"io"
66

77
"github.com/drone/runner-go/pipeline/runtime"
88

99
"github.com/harness/lite-engine/api"
1010
"github.com/harness/lite-engine/engine"
1111
)
1212

13-
func executeRunStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest) (
13+
func executeRunStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer) (
1414
*runtime.State, error) {
1515
step := toStep(r)
1616
step.Command = r.Run.Command
1717
step.Entrypoint = r.Run.Entrypoint
1818

19-
return engine.Run(ctx, step, os.Stdout)
19+
return engine.Run(ctx, step, out)
2020
}

executor/runtest.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1 @@
11
package executor
2-
3-
func executeRunTestStep() {}

executor/step.go

Lines changed: 77 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,17 @@ package executor
22

33
import (
44
"context"
5-
"fmt"
65
"sync"
6+
"time"
77

88
"github.com/harness/lite-engine/api"
99
"github.com/harness/lite-engine/engine"
1010
"github.com/harness/lite-engine/errors"
11+
"github.com/harness/lite-engine/livelog"
12+
"github.com/harness/lite-engine/logstream"
13+
"github.com/harness/lite-engine/pipeline"
14+
"github.com/hashicorp/go-multierror"
15+
"github.com/sirupsen/logrus"
1116

1217
"github.com/drone/runner-go/pipeline/runtime"
1318
)
@@ -50,21 +55,15 @@ func (e *StepExecutor) StartStep(ctx context.Context, r *api.StartStepRequest) e
5055
e.mu.Lock()
5156
_, ok := e.stepStatus[r.ID]
5257
if ok {
58+
e.mu.Unlock()
5359
return nil
5460
}
5561

5662
e.stepStatus[r.ID] = StepStatus{Status: Running}
5763
e.mu.Unlock()
5864

5965
go func() {
60-
var state *runtime.State
61-
var stepErr error
62-
if r.Kind == api.Run {
63-
state, stepErr = executeRunStep(context.Background(), e.engine, r)
64-
} else {
65-
executeRunTestStep()
66-
}
67-
66+
state, stepErr := e.executeStep(r)
6867
status := StepStatus{Status: Complete, State: state, StepErr: stepErr}
6968
e.mu.Lock()
7069
e.stepStatus[r.ID] = status
@@ -93,6 +92,10 @@ func (e *StepExecutor) PollStep(ctx context.Context, r *api.PollStepRequest) (*a
9392

9493
if s.Status == Complete {
9594
e.mu.Unlock()
95+
if s.StepErr != nil {
96+
return &api.PollStepResponse{}, &errors.InternalServerError{Msg: s.StepErr.Error()}
97+
}
98+
9699
return &api.PollStepResponse{
97100
Exited: s.State.Exited,
98101
ExitCode: s.State.ExitCode,
@@ -110,12 +113,75 @@ func (e *StepExecutor) PollStep(ctx context.Context, r *api.PollStepRequest) (*a
110113

111114
status := <-ch
112115
if status.StepErr != nil {
113-
errMsg := fmt.Sprintf("failed to execute step with err: %s", status.StepErr.Error())
114-
return &api.PollStepResponse{}, &errors.InternalServerError{Msg: errMsg}
116+
return &api.PollStepResponse{}, &errors.InternalServerError{Msg: status.StepErr.Error()}
115117
}
116118
return &api.PollStepResponse{
117119
Exited: status.State.Exited,
118120
ExitCode: status.State.ExitCode,
119121
OOMKilled: status.State.OOMKilled,
120122
}, nil
121123
}
124+
125+
func (e *StepExecutor) executeStep(r *api.StartStepRequest) (*runtime.State, error) {
126+
state := pipeline.GetState()
127+
secrets := append(state.GetSecrets(), r.Secrets...)
128+
129+
// Create a log stream for step logs
130+
client := state.GetLogStreamClient()
131+
wc := livelog.New(client, r.LogKey, getNudges())
132+
wr := logstream.NewReplacer(wc, secrets)
133+
go wr.Open() // nolint:errcheck
134+
135+
// if the step is configured as a daemon, it is detached
136+
// from the main process and executed separately.
137+
if r.Detach {
138+
go func() {
139+
ctx := context.Background()
140+
var cancel context.CancelFunc
141+
if r.Timeout > 0 {
142+
ctx, cancel = context.WithTimeout(ctx, time.Second*time.Duration(r.Timeout))
143+
defer cancel()
144+
}
145+
executeRunStep(ctx, e.engine, r, wr) // nolint:errcheck
146+
wc.Close()
147+
}()
148+
return &runtime.State{Exited: false}, nil
149+
}
150+
151+
var result error
152+
153+
ctx := context.Background()
154+
var cancel context.CancelFunc
155+
if r.Timeout > 0 {
156+
ctx, cancel = context.WithTimeout(ctx, time.Second*time.Duration(r.Timeout))
157+
defer cancel()
158+
}
159+
160+
exited, stepErr := executeRunStep(ctx, e.engine, r, wr)
161+
if stepErr != nil {
162+
result = multierror.Append(result, stepErr)
163+
}
164+
165+
// close the stream. If the session is a remote session, the
166+
// full log buffer is uploaded to the remote server.
167+
if err := wc.Close(); err != nil {
168+
result = multierror.Append(result, err)
169+
}
170+
171+
// if the context was canceled and returns a canceled or
172+
// DeadlineExceeded error this indicates the pipeline was
173+
// canceled.
174+
switch ctx.Err() {
175+
case context.Canceled, context.DeadlineExceeded:
176+
return nil, ctx.Err()
177+
}
178+
179+
if exited != nil {
180+
if exited.OOMKilled {
181+
logrus.WithField("id", r.ID).Infoln("received oom kill.")
182+
} else {
183+
logrus.WithField("id", r.ID).Infof("received exit code %d\n", exited.ExitCode)
184+
}
185+
}
186+
return exited, result
187+
}

go.mod

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,16 @@ go 1.17
55
replace github.com/docker/docker => github.com/docker/engine v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
66

77
require (
8+
github.com/cenkalti/backoff/v4 v4.1.1
89
github.com/docker/distribution v2.7.1+incompatible
910
github.com/docker/docker v0.0.1
1011
github.com/docker/go-connections v0.4.0
12+
github.com/drone/drone-go v1.6.0
1113
github.com/drone/runner-go v1.9.0
1214
github.com/go-chi/chi v1.5.4
1315
github.com/gofrs/uuid v4.0.0+incompatible
16+
github.com/google/go-cmp v0.3.0
17+
github.com/hashicorp/go-multierror v1.0.0
1418
github.com/joho/godotenv v1.4.0
1519
github.com/kelseyhightower/envconfig v1.4.0
1620
github.com/pkg/errors v0.9.1
@@ -30,13 +34,11 @@ require (
3034
github.com/containerd/containerd v1.3.4 // indirect
3135
github.com/coreos/go-semver v0.3.0 // indirect
3236
github.com/docker/go-units v0.4.0 // indirect
33-
github.com/drone/drone-go v1.6.0 // indirect
3437
github.com/drone/envsubst v1.0.3 // indirect
3538
github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506 // indirect
3639
github.com/golang/protobuf v1.3.3 // indirect
3740
github.com/gorilla/mux v1.7.4 // indirect
3841
github.com/hashicorp/errwrap v1.0.0 // indirect
39-
github.com/hashicorp/go-multierror v1.0.0 // indirect
4042
github.com/kr/pretty v0.1.0 // indirect
4143
github.com/morikuni/aec v1.0.0 // indirect
4244
github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTS
1515
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
1616
github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8=
1717
github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI=
18+
github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
19+
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
1820
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
1921
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
2022
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=

0 commit comments

Comments
 (0)