diff --git a/.github/workflows/cf-driver-go.yml b/.github/workflows/cf-driver-go.yml index 0e98f31..3cad2b8 100644 --- a/.github/workflows/cf-driver-go.yml +++ b/.github/workflows/cf-driver-go.yml @@ -5,10 +5,6 @@ name: "CF Driver: Go Build & Test" on: [pull_request] -defaults: - run: - working-directory: runner-manager/cfd - jobs: build: runs-on: ubuntu-latest @@ -17,12 +13,9 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 - with: - go-version-file: runner-manager/cfd/go.mod - cache-dependency-path: runner-manager/cfd/go.sum - name: Install dependencies - run: go get . + run: go get ./... - name: Check formatting run: test -z "$(gofmt -l .)" diff --git a/.lazy.lua b/.lazy.lua new file mode 100644 index 0000000..4802c03 --- /dev/null +++ b/.lazy.lua @@ -0,0 +1,21 @@ +return { + { + "nvim-neotest/neotest", + config = function() + ---@diagnostic disable-next-line: missing-fields + require("neotest").setup({ + adapters = { + require("neotest-golang")({ + go_test_args = { "-v", "-race", "-count=1", "-tags=integration" }, + go_list_args = { "-tags=integration" }, + dap_go_opts = { + delve = { + build_flags = { "-tags=integration" }, + }, + }, + }), + }, + }) + end, + }, +} diff --git a/runner-manager/cfd/Makefile b/Makefile similarity index 60% rename from runner-manager/cfd/Makefile rename to Makefile index 28732e0..60bfb5b 100644 --- a/runner-manager/cfd/Makefile +++ b/Makefile @@ -8,10 +8,10 @@ vet: fmt go vet ./... test: vet - go test ./... + go test -v ./... integration: vet - go test -count=1 --tags=integration ./... + go test -v -count=1 --tags=integration ./... build: vet - go build + go build ./runner-manager/cfd diff --git a/runner-manager/cfd/go.mod b/go.mod similarity index 94% rename from runner-manager/cfd/go.mod rename to go.mod index a433e81..2d56d64 100644 --- a/runner-manager/cfd/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/GSA-TTS/gitlab-runner-cloudgov/runner/cfd +module github.com/GSA-TTS/gitlab-runner-cloudgov go 1.23.5 diff --git a/runner-manager/cfd/go.sum b/go.sum similarity index 96% rename from runner-manager/cfd/go.sum rename to go.sum index b25180c..7ada064 100644 --- a/runner-manager/cfd/go.sum +++ b/go.sum @@ -45,5 +45,7 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/runner-manager/cfd/.vscode/launch.json b/runner-manager/cfd/.vscode/launch.json deleted file mode 100644 index ea3fa7e..0000000 --- a/runner-manager/cfd/.vscode/launch.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Debug main.go", - "type": "go", - "request": "launch", - "program": "main.go" - } - ] -} diff --git a/runner-manager/cfd/README.md b/runner-manager/cfd/README.md index 9865c90..90f2395 100644 --- a/runner-manager/cfd/README.md +++ b/runner-manager/cfd/README.md @@ -50,12 +50,14 @@ make test ### Integration tests -We only have one integration test right now, and to get it running you'll need to do a bit of local setup. - -1. You will need to first get a username & password for some space on cloud.gov that has at least one app. -1. Then you can add those credentials to `./cg/testdata/.cg_creds` in the style of the `.cg_creds.sample` file there. -1. Run the test with `make integration`, which should give you an error and, in its output, show you what the resulting JSON looks like. -1. Copy that JSON result over to the last line of your `.cg_creds` file and run `make integration` again, this time it should succeed. +Integration tests take a little effort get working. + +1. Set your `cf target` to `sandbox-gsa`. +2. Run `./sh/integration_setup.sh`. + a. This will output credentials and `cf target` info to the `testdata` directories for `./cloudgov` and `./cmd/drive`. + b. It will also create a sample app in your sandbox account to be used for testing. +3. Run integration tests with `make integration` +4. Whenever you're ready, you can clean up the credentials & app made during setup with `./sh/integration_teardown.sh`. ## Builds diff --git a/runner-manager/cfd/cloudgov/cf_client.go b/runner-manager/cfd/cloudgov/cf_client.go index f516230..0f0b838 100644 --- a/runner-manager/cfd/cloudgov/cf_client.go +++ b/runner-manager/cfd/cloudgov/cf_client.go @@ -73,7 +73,7 @@ func castApp(app *resource.App) *App { if app == nil || app.GUID == "" { return nil } - return &(App{Name: app.Name, GUID: app.GUID, State: app.State}) + return &(App{Name: app.Name, GUID: app.GUID, State: app.State, SpaceGUID: app.Relationships.Space.Data.GUID}) } func castApps(apps []*resource.App) []*App { @@ -104,3 +104,35 @@ func (cf *CFClientAPI) appsList() ([]*App, error) { } return castApps(apps), nil } + +func (cf *CFClientAPI) sshCode() (string, error) { + ctx := context.Background() + return cf.conn().SSHCode(ctx) +} + +func (cf *CFClientAPI) mapRoute( + ctx context.Context, + app *App, + domain string, space string, host string, path string, port int, +) error { + opts := resource.NewRouteCreateWithHost(domain, space, host, path, port) + + route, err := cf.conn().Routes.Create(ctx, opts) + if err != nil { + return err + } + + _, err = cf.conn().Routes.InsertDestinations( + ctx, + route.GUID, + []*resource.RouteDestinationInsertOrReplace{{ + App: resource.RouteDestinationApp{GUID: &app.GUID}, + }}, + ) + return err +} + +// addNetworkPolicy implements ClientAPI. +func (cf *CFClientAPI) addNetworkPolicy(app *App, dest string, space string, port string) error { + panic("unimplemented") +} diff --git a/runner-manager/cfd/cloudgov/cloudgov.go b/runner-manager/cfd/cloudgov/cloudgov.go index 13f27f4..b4b4f75 100644 --- a/runner-manager/cfd/cloudgov/cloudgov.go +++ b/runner-manager/cfd/cloudgov/cloudgov.go @@ -1,18 +1,7 @@ package cloudgov -import ( - "errors" - "fmt" +import "context" - "github.com/cloudfoundry/go-cfclient/v3/resource" -) - -// Stuff we'll need to implement, for ref -// -// mapRoute() -// -// addNetworkPolicy() -// removeNetworkPolicy() type ClientAPI interface { connect(url string, creds *Creds) error @@ -20,6 +9,10 @@ type ClientAPI interface { appPush(m *AppManifest) (*App, error) appDelete(id string) error appsList() (apps []*App, err error) + + sshCode() (string, error) + mapRoute(ctx context.Context, app *App, domain string, space string, host string, path string, port int) error + addNetworkPolicy(app *App, dest string, space string, port string) error } type CredsGetter interface { @@ -46,8 +39,10 @@ func (e CloudGovClientError) Error() string { return e.msg } -// TODO: we should pull this out of VCAP_APPLICATION -const apiRootURLDefault = "https://api.fr.cloud.gov" +const ( + apiRootURLDefault = "https://api.fr-stage.cloud.gov" + internalDomainGUID = "8a5d6a8c-cfc1-4fc4-afc9-aa563ff9df5e" +) func New(i ClientAPI, o *Opts) (*Client, error) { if o == nil { @@ -83,9 +78,10 @@ func (c *Client) Connect() (*Client, error) { } type App struct { - Name string - GUID string - State string + Name string + GUID string + State string + SpaceGUID string } func (c *Client) AppGet(id string) (*App, error) { @@ -102,34 +98,15 @@ func (c *Client) AppsList() ([]*App, error) { // TODO: this abstraction might belong in /cmd, // unless it can be further generalized to all pushes -func (c *Client) ServicePush(manifest *AppManifest) (*App, error) { +func (c *Client) Push(manifest *AppManifest) (*App, error) { containerID := manifest.Name if containerID == "" { - return nil, CloudGovClientError{"ServicePush: AppManifest.Name must be defined"} + return nil, CloudGovClientError{"Push: AppManifest.Name must be defined"} } if manifest.OrgName == "" || manifest.SpaceName == "" { - return nil, CloudGovClientError{"ServicePush: AppManifest must have Org and Space names"} - } - - // check for an old instance of the service, delete if found - app, err := c.AppGet(containerID) - if err != nil { - var cferr resource.CloudFoundryError - if errors.As(err, &cferr) { - err = nil - if cferr.Code != 10010 { - return nil, fmt.Errorf("unexpected cferr checking for existing app: %w", cferr) - } - } else { - return nil, fmt.Errorf("error checking for existing service (%v): %w", containerID, err) - } - } - if app != nil { - if err := c.AppDelete(containerID); err != nil { - return nil, fmt.Errorf("error deleting existing service (%v): %w", containerID, err) - } + return nil, CloudGovClientError{"Push: AppManifest must have Org and Space names"} } return c.appPush(manifest) @@ -144,7 +121,7 @@ func (c *Client) ServicesPush(manifests []*AppManifest) ([]*App, error) { apps := make([]*App, len(manifests)) for i, s := range manifests { - app, err := c.ServicePush(s) + app, err := c.Push(s) if err != nil { return nil, err } @@ -153,3 +130,11 @@ func (c *Client) ServicesPush(manifests []*AppManifest) ([]*App, error) { return apps, nil } + +func (c *Client) SSHCode() (string, error) { + return c.sshCode() +} + +func (c *Client) MapServiceRoute(app *App) error { + return c.mapRoute(context.Background(), app, internalDomainGUID, app.SpaceGUID, app.Name, "", 0) +} diff --git a/runner-manager/cfd/cloudgov/cloudgov_integration_test.go b/runner-manager/cfd/cloudgov/cloudgov_integration_test.go index f175e7b..46ea48f 100644 --- a/runner-manager/cfd/cloudgov/cloudgov_integration_test.go +++ b/runner-manager/cfd/cloudgov/cloudgov_integration_test.go @@ -3,94 +3,59 @@ package cloudgov_test import ( - "bufio" "encoding/json" "errors" "fmt" - "os" + "os/exec" + "regexp" "testing" - cg "github.com/GSA-TTS/gitlab-runner-cloudgov/runner/cfd/cloudgov" + cg "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cloudgov" + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/internal/tutils" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" ) var ( - appGetWanted string - cgClient *cg.Client + cgClient *cg.Client + app, + org, + space string ) -func TestMain(m *testing.M) { - var user, pass string - var err error - - path := "./testdata/.cloudgov_creds" - f, err := os.Open(path) - if err != nil { - fmt.Printf( - "Error opening testdata file = %v\n\033[1;33mDid you forget to create `%v`?\033[0m", - err, path, - ) - os.Exit(1) - } - defer f.Close() - - scanner := bufio.NewScanner(f) - - var i int - var l [3]string - for scanner.Scan() { - text := scanner.Text() - if text[0] == '#' { - continue - } - l[i] = text - if i++; i > 2 { - user, pass, appGetWanted = l[0], l[1], l[2] - break - } - } - - if err = scanner.Err(); err != nil { - fmt.Printf("Error scanning testdata file = %v", err) - return - } - - if user == "" || pass == "" { - fmt.Printf("Could not load variables from testdata") - return - } - - cgClient, err = cg.New(&cg.CFClientAPI{}, &cg.Opts{ - Creds: &cg.Creds{Username: user, Password: pass}, - }) - if err != nil { - fmt.Printf("Error getting cloudgovClient = %v", err) +func setup(t testing.TB) { + t.Helper() + if cgClient != nil { return } + cgClient, org, space, app = tutils.IntegrationSetup(t) +} - m.Run() +func getCmpOpts() cmp.Option { + return cmpopts.IgnoreFields(cg.App{}, "GUID") } func Test_CFAdapter_AppGet(t *testing.T) { - apps, err := cgClient.AppsList() - if err != nil { - t.Errorf("Error running AppsList() = %v", err) - return - } + setup(t) + + want := []*cg.App{{ + Name: app, + State: "STARTED", + }} - got, err := json.Marshal(apps) + got, err := cgClient.AppsList() if err != nil { - t.Errorf("Error marshalling apps to json = %v", err) - return + t.Fatalf("Error running AppsList() = %v", err) } - if diff := cmp.Diff(string(got), appGetWanted); diff != "" { - t.Errorf("mismatch (-got +want):\n%s", diff) - return + if diff := cmp.Diff(got, want, getCmpOpts()); diff != "" { + t.Fatalf("mismatch (-got +want):\n%s", diff) } } -func Test_ServicePush(t *testing.T) { +func Test_Push(t *testing.T) { + setup(t) + tests := map[string]struct { want *cg.App wantErr error @@ -100,12 +65,12 @@ func Test_ServicePush(t *testing.T) { wantErr: errors.New("could not find org bad: expected exactly 1 result, but got less or more than 1"), manifest: &cg.AppManifest{Name: "Fail", OrgName: "bad", SpaceName: "bad"}, }, - "Passes with real org and space": { - want: &cg.App{Name: "c0f91804-4d3a-47df-be14-c9eb4fb59324", State: "STARTED"}, + "Passes with sandbox space": { + want: &cg.App{Name: "Test_Push_App", State: "STARTED"}, manifest: &cg.AppManifest{ - OrgName: "gsa-tts-devtools-prototyping", - SpaceName: "cgd-int", - Name: "Some cool app", + OrgName: org, + SpaceName: space, + Name: "Test_Push_App", Docker: cg.AppManifestDocker{ Image: "busybox", }, @@ -119,14 +84,96 @@ func Test_ServicePush(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { c := cgClient - got, err := c.ServicePush(tt.manifest) + got, err := c.Push(tt.manifest) + + if got != nil && got.GUID != "" { + tutils.CleanupApp(t, c, got.GUID) + } + if err != nil && (tt.wantErr == nil || err.Error() != tt.wantErr.Error()) { - t.Errorf("Client.ServicePush() error = %v, wantErr = %v", err, tt.wantErr) + t.Errorf("Client.Push() error = %v, wantErr = %v", err, tt.wantErr) return } - if diff := cmp.Diff(got, tt.want); diff != "" { + + if diff := cmp.Diff(got, tt.want, getCmpOpts()); diff != "" { t.Errorf("mismatch (-got +want):\n%s", diff) } }) } } + +func Test_SSHCode(t *testing.T) { + setup(t) + got, err := cgClient.SSHCode() + if err != nil { + t.Errorf("got error = %v", err) + return + } + + re := regexp.MustCompile(`[\w-_]{32}`) + if !re.MatchString(got) { + t.Errorf("wanted string matching /%v/, got %v", re, got) + } +} + +func cleanupRoute(t testing.TB, app *cg.App) error { + t.Helper() + + delRouteCmd := exec.Command( + "cf", "delete-route", "-f", "apps.internal", + fmt.Sprintf("-n%s", app.Name), + ) + + out, err := delRouteCmd.CombinedOutput() + if err != nil { + t.Log(string(out)) + if exErr, ok := err.(*exec.ExitError); ok { + t.Log(exErr.Error()) + t.Fatal(string(exErr.Stderr)) + } else { + t.Fatal(err) + } + } + + return err +} + +func TestClient_MapServiceRoute(t *testing.T) { + setup(t) + + apps, err := cgClient.AppsList() + if err != nil { + t.Fatal(err) + } + app := apps[0] + + err = cgClient.MapServiceRoute(app) + defer cleanupRoute(t, app) + if err != nil { + t.Fatal(err) + } + + ckRouteCmd := exec.Command("cf", "curl", fmt.Sprintf("/v3/apps/%s/routes", app.GUID)) + out, err := ckRouteCmd.CombinedOutput() + if err != nil { + t.Log(out) + t.Fatal(err) + } + + var routeOut map[string][]map[string]string + if err := json.Unmarshal(out, &routeOut); err != nil { + t.Log("partial unmarshalling error expected…") + t.Log(err) + } + + wantUrl := fmt.Sprintf("%s.apps.internal", app.Name) + + for _, m := range routeOut["resources"] { + if m["host"] == app.Name && m["url"] == wantUrl { + return + } + } + + t.Logf("%#v", routeOut["resources"]) + t.Fatalf("could not find route with %s host and correct url", app.Name) +} diff --git a/runner-manager/cfd/cloudgov/cloudgov_test.go b/runner-manager/cfd/cloudgov/cloudgov_test.go index f321036..0438f55 100644 --- a/runner-manager/cfd/cloudgov/cloudgov_test.go +++ b/runner-manager/cfd/cloudgov/cloudgov_test.go @@ -335,7 +335,7 @@ func TestClient_AppsList(t *testing.T) { } } -func TestClient_ServicePush(t *testing.T) { +func TestClient_Push(t *testing.T) { optsStub := &Opts{CredsGetter: stubCredsGetter{"a", "b", false}} cgStub := &Client{&stubClientAPI{ StURL: apiRootURLDefault, @@ -359,12 +359,12 @@ func TestClient_ServicePush(t *testing.T) { "Fails without name": { fields: fields{ClientAPI: cgStub, Opts: optsStub}, args: args{manifest: &AppManifest{}}, - wantErr: CloudGovClientError{"ServicePush: AppManifest.Name must be defined"}, + wantErr: CloudGovClientError{"Push: AppManifest.Name must be defined"}, }, "Fails without org": { fields: fields{ClientAPI: cgStub, Opts: optsStub}, args: args{manifest: &AppManifest{Name: "Some App"}}, - wantErr: CloudGovClientError{"ServicePush: AppManifest must have Org and Space names"}, + wantErr: CloudGovClientError{"Push: AppManifest must have Org and Space names"}, }, "Passes with all fields": { fields: fields{ClientAPI: cgStub, Opts: optsStub}, @@ -379,12 +379,12 @@ func TestClient_ServicePush(t *testing.T) { ClientAPI: tt.fields.ClientAPI, Opts: tt.fields.Opts, } - got, err := c.ServicePush(tt.args.manifest) + got, err := c.Push(tt.args.manifest) if err != nil || tt.wantErr != nil { if tt.wantErr == nil { t.Errorf("Client.AppsList() error = %v", err) } else if diff := cmp.Diff(tt.wantErr, err, cmpopts.EquateErrors()); diff != "" { - t.Errorf("Client.ServicePush() error mismatch (-want +got):\n%s", diff) + t.Errorf("Client.Push() error mismatch (-want +got):\n%s", diff) } return } diff --git a/runner-manager/cfd/cloudgov/testdata/.cloudgov_creds.sample b/runner-manager/cfd/cloudgov/testdata/.cloudgov_creds.sample deleted file mode 100644 index 862cd3f..0000000 --- a/runner-manager/cfd/cloudgov/testdata/.cloudgov_creds.sample +++ /dev/null @@ -1,11 +0,0 @@ -# For simple integration test with cloudgov_integration_test.go -# -# This test will use the credentials provided to run `cf apps` and -# check the output against what you have provided. -# -# 1. copy this file to `.cloudgov_creds` -# 2. replace with real credentials, e.g. a service key's -# 3. replace with real output from `cf apps` -username-1234-asdf -password-asdf-1234 -I am the expected output! diff --git a/runner-manager/cfd/cmd/drive/get_job_config.go b/runner-manager/cfd/cmd/drive/get_job_config.go index 1336592..c3a6ff4 100644 --- a/runner-manager/cfd/cmd/drive/get_job_config.go +++ b/runner-manager/cfd/cmd/drive/get_job_config.go @@ -6,9 +6,10 @@ import ( "os" "reflect" "regexp" + "slices" "strings" - "github.com/GSA-TTS/gitlab-runner-cloudgov/runner/cfd/cloudgov" + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cloudgov" ) type JobConfig struct { @@ -18,6 +19,12 @@ type JobConfig struct { VcapAppData VcapAppJSON string `env:"VCAP_APPLICATION"` + VcapServicesData + VcapServicesJSON string `env:"VCAP_SERVICES"` + EgressServiceName string `env:"PROXY_CREDENTIAL_INSTANCE"` + + EgressProxyConfig + Manifest *cloudgov.AppManifest // We combine the following to make the container ID. @@ -65,10 +72,34 @@ type VcapAppData struct { CFApi string `json:"cf_api"` OrgID string `json:"org_id"` OrgName string `json:"organization_name"` - SpaceId string `json:"space_id"` + SpaceID string `json:"space_id"` SpaceName string `json:"space_name"` } +type ( + VcapServicesData map[string][]VcapServiceInstance + VcapServiceInstance struct { + Name string + Credentials VcapServiceCredentials + } +) + +type VcapServiceCredentials struct { + Domain string `json:"domain"` + HTTPPort int `json:"http_port"` + HTTPURI string `json:"http_uri"` + HTTPSURI string `json:"https_uri"` + CredString string `json:"cred_string"` +} + +type EgressProxyConfig struct { + ProxyHostHTTP string + ProxyHostHTTPS string + ProxyHostSSH string + ProxyPortSSH int + ProxyAuthFile string +} + func parseCfgJSON[R any](j []byte, r *R) (*R, error) { if len(j) < 1 { return r, nil @@ -112,6 +143,13 @@ func (c *JobConfig) parseVcapAppJSON() (err error) { return err } +func (c *JobConfig) parseVcapServicesJSON() (err error) { + ref := &map[string][]VcapServiceInstance{} + ref, err = parseCfgJSON([]byte(c.VcapServicesJSON), ref) + c.VcapServicesData = *ref + return err +} + // This is a pretty simple implementation, if our needs get more // complex we should use one of several existing packages to do this. // e.g., https://pkg.go.dev/github.com/caarlos0/env/v11 @@ -182,6 +220,42 @@ func (cfg *JobConfig) processImage(img Image, m *cloudgov.AppManifest) { } } +func (cfg *JobConfig) processEgressProxyCfg() (err error) { + defer (func() { + if err != nil { + err = fmt.Errorf("error processEgressProxyCfg: %w", err) + } + })() + + userServices := cfg.VcapServicesData["user-provided"] + if len(userServices) < 1 { + return nil + } + + egressIdx := slices.IndexFunc(userServices, func(vsi VcapServiceInstance) bool { + return vsi.Name == cfg.EgressServiceName + }) + if egressIdx < 0 { + return nil + } + + esc := userServices[egressIdx].Credentials + + cfg.EgressProxyConfig = EgressProxyConfig{ + ProxyHostHTTP: esc.HTTPURI, + ProxyHostHTTPS: esc.HTTPSURI, + ProxyHostSSH: esc.Domain, + ProxyPortSSH: esc.HTTPPort, + ProxyAuthFile: os.Getenv("PROXY_AUTH_FILE"), + } + + if cfg.ProxyAuthFile == "" { + cfg.ProxyAuthFile = "/home/vcap/app/ssh_proxy.auth" + } + + return os.WriteFile(cfg.ProxyAuthFile, []byte(esc.CredString), 0600) +} + func getJobConfig() (cfg *JobConfig, err error) { defer func() { if err != nil { @@ -197,6 +271,12 @@ func getJobConfig() (cfg *JobConfig, err error) { if err = cfg.parseVcapAppJSON(); err != nil { return nil, err } + if err = cfg.parseVcapServicesJSON(); err != nil { + return nil, err + } + if err = cfg.processEgressProxyCfg(); err != nil { + return nil, err + } cfg.ContainerID = fmt.Sprintf( "glrw-p%v-c%v-j%v", diff --git a/runner-manager/cfd/cmd/drive/get_job_config_test.go b/runner-manager/cfd/cmd/drive/get_job_config_test.go index 716083f..7a71ecb 100644 --- a/runner-manager/cfd/cmd/drive/get_job_config_test.go +++ b/runner-manager/cfd/cmd/drive/get_job_config_test.go @@ -1,9 +1,11 @@ package drive import ( + "os" + "path/filepath" "testing" - "github.com/GSA-TTS/gitlab-runner-cloudgov/runner/cfd/cloudgov" + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cloudgov" "github.com/google/go-cmp/cmp" ) @@ -12,16 +14,17 @@ import ( // think about that later. func Test_GetJobConfig(t *testing.T) { cfgWant := &JobConfig{ - JobResponse: JobResponse{}, - CIRegistryUser: "foo", - CIRegistryPass: "bar", - DockerHubUser: "foo", - DockerHubToken: "1234", - WorkerMemory: "1024M", - WorkerDiskSize: "1024M", - JobResponseFile: "", - VcapAppJSON: "", - ContainerID: "glrw-p-c-j", + JobResponse: JobResponse{}, + CIRegistryUser: "foo", + CIRegistryPass: "bar", + DockerHubUser: "foo", + DockerHubToken: "1234", + WorkerMemory: "1024M", + WorkerDiskSize: "1024M", + JobResponseFile: "", + VcapAppJSON: "", + VcapServicesData: VcapServicesData{}, + ContainerID: "glrw-p-c-j", Manifest: &cloudgov.AppManifest{ Name: "glrw-p-c-j", NoRoute: true, @@ -50,7 +53,7 @@ func Test_GetJobConfig(t *testing.T) { t.Error(err) return } - if diff := cmp.Diff(cfgWant, parsedCfg); diff != "" { + if diff := cmp.Diff(parsedCfg, cfgWant); diff != "" { t.Error(diff) } } @@ -80,8 +83,9 @@ func Test_parseJobResponseFile(t *testing.T) { Process: cloudgov.AppManifestProcess{Command: "j k l g h i", HealthCheckType: "process"}, }, Config: &JobConfig{ - ContainerID: "glrw-p-c-j", - JobResponseFile: "./testdata/sample_job_response.json", + ContainerID: "glrw-p-c-j", + JobResponseFile: "./testdata/sample_job_response.json", + VcapServicesData: VcapServicesData{}, Manifest: &cloudgov.AppManifest{ Name: "glrw-p-c-j", Env: map[string]string{"foo": "bar"}, @@ -114,7 +118,7 @@ func Test_parseVcapAppJSON(t *testing.T) { wanted := VcapAppData{ CFApi: "https://api.fr.cloud.gov", OrgName: "gsa-tts-devtools-prototyping", - SpaceId: "8969a4b6-01aa-431d-9790-77cc4c47e3e7", + SpaceID: "8969a4b6-01aa-431d-9790-77cc4c47e3e7", SpaceName: "zjr-gl-test", } @@ -128,3 +132,69 @@ func Test_parseVcapAppJSON(t *testing.T) { t.Errorf("mismatch (-got +want):\n%s", diff) } } + +func Test_parseVcapServicesJSON(t *testing.T) { + sample := `{"s3":[{"label":"s3","provider":null,"plan":"basic-sandbox","name":"glr-dependency-cache","tags":["AWS","S3","object-storage","terraform-cloudgov-managed"],"instance_guid":"d1541026","instance_name":"glr-dependency-cache","binding_guid":"9f316c56","binding_name":null,"credentials":{"uri":"s3://goooo:booo@s3-fips.us-gov-west-1.aaws.com/cg-d1541026","insecure_skip_verify":false,"access_key_id":"jjjjj","secret_access_key":"ssssssss","region":"us-gov-west-1","bucket":"cg-d1541026","endpoint":"s3-fips.us-gov-west-1.amazonaws.com","fips_endpoint":"s3-fips.us-gov-west-1.amazonaws.com","additional_buckets":[]},"syslog_drain_url":null,"volume_mounts":[]}],"user-provided":[{"label":"user-provided","name":"glr-egress-proxy-credentials","tags":[],"instance_guid":"608e3f73","instance_name":"glr-egress-proxy-credentials","binding_guid":"7530ea7b","binding_name":null,"credentials":{"cred_string":"bingo:dingo","domain":"egress-proxy.apps.internal","http_port":8080,"http_uri":"http://bingo:dingo@egress-proxy.apps.internal:8080","https_uri":"https://bingo:dingo@egress-proxy.apps.internal:61443"},"syslog_drain_url":null,"volume_mounts":[]}]}` + t.Setenv("VCAP_SERVICES", sample) + t.Setenv("PROXY_CREDENTIAL_INSTANCE", "glr-egress-proxy-credentials") + + dir, err := os.MkdirTemp("", "temp_auth_files") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + authFile := filepath.Join(dir, "ssh_proxy.auth") + t.Setenv("PROXY_AUTH_FILE", authFile) + + credStringWanted := "bingo:dingo" + + wantedServices := VcapServicesData{ + "s3": []VcapServiceInstance{{ + Name: "glr-dependency-cache", + }}, + "user-provided": []VcapServiceInstance{{ + Name: "glr-egress-proxy-credentials", + Credentials: VcapServiceCredentials{ + Domain: "egress-proxy.apps.internal", + HTTPPort: 8080, + HTTPURI: "http://bingo:dingo@egress-proxy.apps.internal:8080", + HTTPSURI: "https://bingo:dingo@egress-proxy.apps.internal:61443", + CredString: credStringWanted, + }, + }}, + } + + wantedEgressConfig := EgressProxyConfig{ + ProxyHostHTTP: "http://bingo:dingo@egress-proxy.apps.internal:8080", + ProxyHostHTTPS: "https://bingo:dingo@egress-proxy.apps.internal:61443", + ProxyHostSSH: "egress-proxy.apps.internal", + ProxyPortSSH: 8080, + ProxyAuthFile: authFile, + } + + cfg, err := getJobConfig() + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(cfg.VcapServicesData, wantedServices); diff != "" { + t.Errorf("mismatch (-got +want):\n%s", diff) + } + + if diff := cmp.Diff(cfg.VcapServicesData["user-provided"][0], wantedServices["user-provided"][0]); diff != "" { + t.Fatalf("mismatch (-got +want):\n%s", diff) + } + + if diff := cmp.Diff(cfg.EgressProxyConfig, wantedEgressConfig); diff != "" { + t.Fatalf("mismatch (-got +want):\n%s", diff) + } + + credString, err := os.ReadFile(cfg.ProxyAuthFile) + if err != nil { + t.Fatalf("error reading ProxyAuthFile: %v", err) + } + if diff := cmp.Diff(string(credString), credStringWanted); diff != "" { + t.Fatalf("mismatch (-got +want):\n%s", diff) + } +} diff --git a/runner-manager/cfd/cmd/drive/prepare.go b/runner-manager/cfd/cmd/drive/prepare.go index 69d27cb..ccb6220 100644 --- a/runner-manager/cfd/cmd/drive/prepare.go +++ b/runner-manager/cfd/cmd/drive/prepare.go @@ -28,31 +28,44 @@ job log. Read more in GitLab's documentation: https://docs.gitlab.com/runner/executors/custom.html#prepare`, - Run: run, + RunE: run, } type prepStage commonStage -func run(cmd *cobra.Command, args []string) { - // Move this stuff into a setup, add methods. - s, err := newStage() +func run(cmd *cobra.Command, args []string) error { + s, err := newStage(nil) if err != nil { - panic(fmt.Errorf("error getting cgClient: %w", err)) + return fmt.Errorf("error initializing prepare stage: %w", err) } - s.prep.startServices() - - // if services, start services + err = s.prep.exec() + if err != nil { + return fmt.Errorf("error executing prepare stage: %w", err) + } - // if os.Getenv("") + return nil +} - // create temp manifest +func (s *prepStage) exec() (err error) { + // Looping service manifests to run `cf push` + err = s.startServices() + if err != nil { + return err + } - // start container + // Pushing the main job config pulled from get_job_config.go + _, err = s.client.Push(s.config.Manifest) + if err != nil { + return err + } - // install deps + err = s.installDeps() + if err != nil { + return err + } - // allow access to services + return s.setNetworkPolicies() } // TODO: refactor to include a service manifests slice and @@ -63,16 +76,23 @@ func (s *prepStage) startServices() error { } for _, serv := range s.config.Services { - s.client.ServicePush(serv.Manifest) - // add docker user/pass - // - // push - // + s.client.Push(serv.Manifest) // map-route containerID apps.internal --hostname containerID - // + + // TODO: implement WSR_ vars + // Leaving this until more is implemented so the form can fit function // export WSR_SERVICE_HOST_$alias=$containerID.apps.internal - // } return nil } + +// TODO: implement +func (s *prepStage) installDeps() error { + panic("unimplemented") +} + +// TODO: implement +func (s *prepStage) setNetworkPolicies() error { + panic("unimplemented") +} diff --git a/runner-manager/cfd/cmd/drive/stage.go b/runner-manager/cfd/cmd/drive/stage.go index 84ae457..91373c5 100644 --- a/runner-manager/cfd/cmd/drive/stage.go +++ b/runner-manager/cfd/cmd/drive/stage.go @@ -2,8 +2,10 @@ package drive import ( "fmt" + "os/exec" + "strings" - "github.com/GSA-TTS/gitlab-runner-cloudgov/runner/cfd/cloudgov" + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cloudgov" ) type stage struct { @@ -16,27 +18,38 @@ type stage struct { } type commonStage struct { + *stage client *cloudgov.Client config *JobConfig } -func newStage() (s *stage, err error) { +func newStage(client *cloudgov.Client) (s *stage, err error) { defer func() { if err != nil { err = fmt.Errorf("error creating stage: %w", err) } }() - s.common.client, err = cloudgov.New(&cloudgov.CFClientAPI{}, nil) - if err != nil { - return - } + s = &stage{} + s.common.stage = s s.common.config, err = getJobConfig() if err != nil { return } + if client != nil { + s.common.client = client + } else { + s.common.client, err = cloudgov.New( + &cloudgov.CFClientAPI{}, + &cloudgov.Opts{APIRootURL: s.common.config.CFApi}, + ) + if err != nil { + return + } + } + // conf s.prep = (*prepStage)(&s.common) // run @@ -44,3 +57,35 @@ func newStage() (s *stage, err error) { return } + +func (s *stage) RunSSH(guid string, cmd string) error { + pass, err := s.common.client.SSHCode() + if err != nil { + return err + } + + args := []string{"ssh", "-p 2222", "-T", "-o StrictHostKeyChecking=no"} + host := fmt.Sprintf("cf:%s/0@ssh.fr-stage.cloud.gov", guid) + + // TODO: can we rely on the Bash runner's .profile's edits to SSH Config? + // See: https://github.com/GSA-TTS/gitlab-runner-cloudgov/issues/136 + epCfg := s.common.config.EgressProxyConfig + if epCfg != (EgressProxyConfig{}) { + proxy := fmt.Sprintf("-o ProxyCommand corkscrew %v %v %%h %%p %v", + epCfg.ProxyHostSSH, epCfg.ProxyPortSSH, epCfg.ProxyAuthFile, + ) + args = append(args, proxy) + } + + sshCmd := exec.Command("sshpass", append(args, host)...) + sshCmd.Stdin = strings.NewReader(pass) // give pass to sshpass through stdin + + out, err := sshCmd.Output() + if err != nil { + return err + } + + fmt.Print(string(out)) + + return nil +} diff --git a/runner-manager/cfd/cmd/drive/stage_integration_test.go b/runner-manager/cfd/cmd/drive/stage_integration_test.go new file mode 100644 index 0000000..e0cc01a --- /dev/null +++ b/runner-manager/cfd/cmd/drive/stage_integration_test.go @@ -0,0 +1,49 @@ +//go:build integration + +package drive + +import ( + "os/exec" + "testing" + + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cloudgov" + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/internal/tutils" +) + +var ( + cgClient *cloudgov.Client + app, + org, + space string +) + +func setup(t testing.TB) { + t.Helper() + if cgClient != nil { + return + } + cgClient, org, space, app = tutils.IntegrationSetup(t) +} + +func Test_RunSSH(t *testing.T) { + setup(t) + + stage, err := newStage(cgClient) + if err != nil { + t.Fatal(err) + } + + apps, err := cgClient.AppsList() + if err != nil { + t.Fatal(err) + } + + err = stage.RunSSH(apps[0].GUID, "echo $VCAP_APPLICATION") + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + t.Fatal(string(exitErr.Stderr)) + } else { + t.Fatal(err) + } + } +} diff --git a/runner-manager/cfd/cmd/drive/testdata/sample_vcap_application.json b/runner-manager/cfd/cmd/drive/testdata/sample_vcap_application.json new file mode 100644 index 0000000..757c5e9 --- /dev/null +++ b/runner-manager/cfd/cmd/drive/testdata/sample_vcap_application.json @@ -0,0 +1,17 @@ +{ + "cf_api": "https://api.fr-stage.cloud.gov", + "limits": { "fds": 16384, "mem": 128, "disk": 8 }, + "application_name": "cfd_integration_test_AppGet_2", + "application_uris": [], + "name": "cfd_integration_test_AppGet_2", + "space_name": "zachary.rollyson", + "space_id": "a9ec0873-e8a5-4d39-99d2-4a934f0ea23d", + "organization_id": "da8e238d-c24a-4528-95db-ae11e76bb8f1", + "organization_name": "sandbox-gsa", + "uris": [], + "process_id": "5d5bea40-e3ba-49a6-b46a-1645b0eac86b", + "process_type": "web", + "application_id": "5d5bea40-e3ba-49a6-b46a-1645b0eac86b", + "version": "141686d8-4b7a-4f27-8cde-8d698c3ef750", + "application_version": "141686d8-4b7a-4f27-8cde-8d698c3ef750" +} diff --git a/runner-manager/cfd/cmd/root.go b/runner-manager/cfd/cmd/root.go index c535dc9..24e6021 100644 --- a/runner-manager/cfd/cmd/root.go +++ b/runner-manager/cfd/cmd/root.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/GSA-TTS/gitlab-runner-cloudgov/runner/cfd/cmd/drive" + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cmd/drive" "github.com/spf13/cobra" ) diff --git a/runner-manager/cfd/internal/tutils/integration_setup.go b/runner-manager/cfd/internal/tutils/integration_setup.go new file mode 100644 index 0000000..d59e471 --- /dev/null +++ b/runner-manager/cfd/internal/tutils/integration_setup.go @@ -0,0 +1,86 @@ +package tutils + +import ( + "bufio" + "fmt" + "os" + "path" + "testing" + + cg "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cloudgov" +) + +const credPath = "./testdata/.cloudgov_creds" + +func IntegrationSetup(t testing.TB) (client *cg.Client, org string, space string, app string) { + var err error + var user, pass string + + defer (func() { + if err != nil { + t.Fatal(fmt.Errorf("IntegrationSetup: %w", err)) + } + })() + + cwd, err := os.Getwd() + if err != nil { + err = fmt.Errorf("getting cwd: %w", err) + return + } + + f, err := os.Open(path.Join(cwd, credPath)) + if err != nil { + err = fmt.Errorf( + "error opening testdata file = %v\n\033[1;33mDid you forget to create `%v`?\033[0m", + err, credPath, + ) + return + } + defer f.Close() + + vars := []*string{&user, &pass, &org, &space, &app} + scanner := bufio.NewScanner(f) + for scanner.Scan() { + text := scanner.Text() + + // Skipping comments + if text[0] == '#' { + continue + } + + l := len(vars) + if l != 0 { + *vars[0] = text + vars = vars[1:] + } else { + break + } + } + + if err = scanner.Err(); err != nil { + err = fmt.Errorf("scanning testdata file: %w", err) + return + } + + if user == "" || pass == "" { + err = fmt.Errorf("could not load variables from testdata") + return + } + + client, err = cg.New(&cg.CFClientAPI{}, &cg.Opts{ + Creds: &cg.Creds{Username: user, Password: pass}, + }) + if err != nil { + err = fmt.Errorf("getting cloudgovClient: %w", err) + return + } + + return client, org, space, app +} + +func CleanupApp(t testing.TB, c *cg.Client, guid string) { + t.Helper() + if err := c.AppDelete(guid); err != nil { + t.Errorf("failed to delete app: %s", guid) + } +} diff --git a/runner-manager/cfd/main.go b/runner-manager/cfd/main.go index 9d780f7..c919d3c 100644 --- a/runner-manager/cfd/main.go +++ b/runner-manager/cfd/main.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/GSA-TTS/gitlab-runner-cloudgov/runner/cfd/cmd" + "github.com/GSA-TTS/gitlab-runner-cloudgov/runner-manager/cfd/cmd" "github.com/joho/godotenv" ) diff --git a/runner-manager/cfd/sh/integration_setup.sh b/runner-manager/cfd/sh/integration_setup.sh new file mode 100755 index 0000000..f0c674a --- /dev/null +++ b/runner-manager/cfd/sh/integration_setup.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +cf_api="api.fr-stage.cloud.gov" +cf_api_prod="api.fr.cloud.gov" +skip_create= + +basename="wsr-integration" +app_name="cfd_integration_test_AppGet" + +usage() { + msg="$1" + status=0 + if [[ -n "$msg" ]]; then + printf "ERROR: %s\n\n" "$msg" >&2 + status=1 + fi + + cat >&2 <<-EOM + Usage: $0 [-bps] + + Creates a service account with key and a sample application, outputs to testdata dirs. + + Options: + -b Basename to use for service account & key (defaults to $basename) + -p Use $cf_api_prod (defaults to $cf_api) + -s Skip creation and only get & output credentials + EOM + + exit $status +} + +dir=$(dirname "$0") +cd "$dir" + +while getopts ":b:psh" opt; do + case $opt in + b) + basename="$OPTARG" + ;; + p) + cf_api="$cf_api_prod" + ;; + s) + skip_create="true" + ;; + h) + usage + ;; + \?) + usage "unknown option '-$OPTARG'" + ;; + esac +done + +set -euo pipefail + +# check login +cf spaces &>/dev/null || cf login -a "$cf_api" --sso + +org=$(cf t | grep org | awk '{print $2}') +if [[ $org != 'sandbox-gsa' ]]; then + echo "ERROR: you should really probably use this in your sandbox for now" + exit 1 +fi + +# create the space deployer, then a key for the deployer +if [[ -z "$skip_create" ]]; then + cf create-service cloud-gov-service-account space-deployer "$basename"-deployer + cf create-service-key "$basename"-deployer "$basename"-key +fi + +# create a teeny app we can use to test client.AppGet +cf push --no-route -k 8M -m 128M -o busybox -u process -c /bin/sh "$app_name" + +out_arr=( + # get the credentials from key and output + "$(cf service-key "$basename"-deployer "$basename"-key | tail +2 | + jq -r ".credentials | .username,.password")" + # get target org & space + "$(cf t | tail -2 | awk '{print $2}')" + "$app_name" +) + +out_str=$( + IFS=$'\n' + echo "${out_arr[*]}" +) + +echo "$out_str" >../cloudgov/testdata/.cloudgov_creds +echo "$out_str" >../cmd/drive/testdata/.cloudgov_creds diff --git a/runner-manager/cfd/sh/integration_teardown.sh b/runner-manager/cfd/sh/integration_teardown.sh new file mode 100755 index 0000000..924c926 --- /dev/null +++ b/runner-manager/cfd/sh/integration_teardown.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +cf_api="api.fr-stage.cloud.gov" +cf_api_prod="api.fr.cloud.gov" + +basename="wsr-integration" +app_name="cfd_integration_test_AppGet" + +usage() { + msg="$1" + status=0 + if [[ -n "$msg" ]]; then + printf "ERROR: %s\n\n" "$msg" >&2 + status=1 + fi + + cat >&2 <<-EOM + Usage: $0 [-bpf] + + Deletes testing service account, key and sample application. + + Options: + -b Basename to use for service account & key (defaults to $basename) + -p Use $cf_api_prod (defaults to $cf_api) + -f Force deletion without confirmation + EOM + + exit $status +} + +declare -a args + +while getopts ":b:pfh" opt; do + case $opt in + b) + basename="$OPTARG" + ;; + p) + cf_api="$cf_api_prod" + ;; + f) + args+=("-f") + ;; + h) + usage + ;; + \?) + usage "unknown option '-$OPTARG'" + ;; + esac +done + +set -euo pipefail + +# check login +cf spaces &>/dev/null || cf login -a "$cf_api" --sso + +org=$(cf t | grep org | awk '{print $2}') +if [[ $org != 'sandbox-gsa' ]]; then + echo "ERROR: you should really probably use this in your sandbox for now" + exit 1 +fi + +# delete the teeny app +cf delete -r "${args[@]}" "$app_name" + +# delete the deployer and key +cf delete-service-key "${args[@]}" "$basename"-deployer "$basename"-key +cf delete-service "${args[@]}" "$basename"-deployer + +echo "WARNING: didn't delete any testdata files, you must remove them manually."