Feature/allow remote href blobs

cmd/add_blob.go

@@ -1,6 +1,9 @@
 package cmd
 
 import (
+	"io"
+	"net/http"
+	"net/url"
 	"os"
 
 	bosherr "github.com/cloudfoundry/bosh-utils/errors"
@@ -22,14 +25,28 @@ func NewAddBlobCmd(blobsDir boshreldir.BlobsDir, fs boshsys.FileSystem, ui boshu
 }
 
 func (c AddBlobCmd) Run(opts AddBlobOpts) error {
-	file, err := c.fs.OpenFile(opts.Args.Path, os.O_RDONLY, 0)
-	if err != nil {
-		return bosherr.WrapErrorf(err, "Opening blob")
+	var file io.ReadCloser
+	var err error
+	href := ""
+	if u, err := url.ParseRequestURI(opts.Args.Path); err == nil && u.Scheme != "" && u.Host != "" {
+		resp, err := http.Get(opts.Args.Path)
+		if err != nil {
+			return bosherr.WrapErrorf(err, "Downloading blob")
+		}
+		defer resp.Body.Close() //nolint:errcheck
+		file = resp.Body
+		href = opts.Args.Path
+	} else {
+		file, err = c.fs.OpenFile(opts.Args.Path, os.O_RDONLY, 0)
+		if err != nil {
+			return bosherr.WrapErrorf(err, "Opening blob")
+		}
+		defer file.Close() //nolint:errcheck
 	}
 
 	defer file.Close() //nolint:errcheck
 
-	blob, err := c.blobsDir.TrackBlob(opts.Args.BlobsPath, file)
+	blob, err := c.blobsDir.TrackBlob(opts.Args.BlobsPath, file, href)
 	if err != nil {
 		return bosherr.WrapErrorf(err, "Tracking blob")
 	}

cmd/add_blob_test.go

@@ -55,8 +55,9 @@ var _ = Describe("AddBlobCmd", func() {
 
 			Expect(blobsDir.TrackBlobCallCount()).To(Equal(1))
 
-			blobsPath, src := blobsDir.TrackBlobArgsForCall(0)
+			blobsPath, src, href := blobsDir.TrackBlobArgsForCall(0)
 			Expect(blobsPath).To(Equal("my-blob.tgz"))
+			Expect(href).To(BeEmpty())
 
 			file := src.(*fakesys.FakeFile)
 			Expect(file.Name()).To(Equal("/path/to/blob.tgz"))

cmd/blobs.go

@@ -29,6 +29,7 @@ func (c BlobsCmd) Run() error {
 			boshtbl.NewHeader("Size"),
 			boshtbl.NewHeader("Blobstore ID"),
 			boshtbl.NewHeader("Digest"),
+			boshtbl.NewHeader("HREF"),
 		},
 
 		SortBy: []boshtbl.ColumnSort{
@@ -48,6 +49,7 @@ func (c BlobsCmd) Run() error {
 			boshtbl.NewValueBytes(uint64(blob.Size)),
 			boshtbl.NewValueString(blobID),
 			boshtbl.NewValueString(blob.SHA1),
+			boshtbl.NewValueString(blob.HREF),
 		})
 	}
 

cmd/blobs_test.go

@@ -44,6 +44,8 @@ var _ = Describe("BlobsCmd", func() {
 
 					BlobstoreID: "",
 					SHA1:        "fake-sha2",
+
+					HREF: "https://somewebsite/path/to/package.tar.gz",
 				},
 			}
 
@@ -60,6 +62,7 @@ var _ = Describe("BlobsCmd", func() {
 					boshtbl.NewHeader("Size"),
 					boshtbl.NewHeader("Blobstore ID"),
 					boshtbl.NewHeader("Digest"),
+					boshtbl.NewHeader("HREF"),
 				},
 
 				SortBy: []boshtbl.ColumnSort{{Column: 0, Asc: true}},
@@ -70,12 +73,14 @@ var _ = Describe("BlobsCmd", func() {
 						boshtbl.NewValueBytes(100),
 						boshtbl.NewValueString("fake-blob-id"),
 						boshtbl.NewValueString("fake-sha1"),
+						boshtbl.NewValueString(""),
 					},
 					{
 						boshtbl.NewValueString("dir/fake-path"),
 						boshtbl.NewValueBytes(1000),
 						boshtbl.NewValueString("(local)"),
 						boshtbl.NewValueString("fake-sha2"),
+						boshtbl.NewValueString("https://somewebsite/path/to/package.tar.gz"),
 					},
 				},
 			}))

cmd/create_release.go

@@ -43,6 +43,13 @@ func (c CreateReleaseCmd) Run(opts CreateReleaseOpts) (boshrel.Release, error) {
 			return nil, err
 		}
 	} else {
+		if opts.ValidateBlobOrigin {
+			err := releaseDir.ValidateBlobsFromOrigin()
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		release, err = c.buildRelease(releaseDir, opts)
 		if err != nil {
 			return nil, err

cmd/opts/opts.go

@@ -1114,9 +1114,10 @@ type CreateReleaseOpts struct {
 	Version          VersionArg `long:"version"            description:"Custom release version (e.g.: 1.0.0, 1.0-beta.2+dev.10)"`
 	TimestampVersion bool       `long:"timestamp-version"  description:"Create release with the timestamp as the dev version (e.g.: 1+dev.TIMESTAMP)"`
 
-	Final   bool    `long:"final"   description:"Make it a final release"`
-	Tarball FileArg `long:"tarball" description:"Create release tarball at path (e.g. /tmp/release.tgz)"`
-	Force   bool    `long:"force"   description:"Ignore Git dirty state check"`
+	Final              bool    `long:"final"                description:"Make it a final release"`
+	Tarball            FileArg `long:"tarball"              description:"Create release tarball at path (e.g. /tmp/release.tgz)"`
+	Force              bool    `long:"force"                description:"Ignore Git dirty state check"`
+	ValidateBlobOrigin bool    `long:"validate-blob-origin" description:"Validate blob SHA with origin"`
 
 	cmd
 }

releasedir/fs_blobs_dir.go

@@ -3,6 +3,7 @@ package releasedir
 import (
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"sort"
@@ -47,6 +48,7 @@ type fsBlobsDirSchema_Blob struct {
 
 	BlobstoreID string `yaml:"object_id,omitempty"`
 	SHA1        string `yaml:"sha"`
+	HREF        string `yaml:"href,omitempty"`
 }
 
 func NewFSBlobsDir(
@@ -106,6 +108,7 @@ func (d FSBlobsDir) Blobs() ([]Blob, error) {
 			Size:        rec.Size,
 			BlobstoreID: rec.BlobstoreID,
 			SHA1:        rec.SHA1,
+			HREF:        rec.HREF,
 		})
 	}
 
@@ -139,10 +142,11 @@ func (d FSBlobsDir) SyncBlobs(parallel int) error {
 
 	var tasks []func() error
 	for _, blob := range blobs {
-		blob := blob
 		tasks = append(tasks, func() error {
 			if len(blob.BlobstoreID) > 0 {
 				return d.downloadBlob(blob)
+			} else if len(blob.HREF) > 0 {
+				return d.downloadBlobFromHref(blob)
 			}
 
 			return nil
@@ -157,6 +161,53 @@ func (d FSBlobsDir) SyncBlobs(parallel int) error {
 	return nil
 }
 
+func (d FSBlobsDir) ValidateBlobsFromOrigin() error {
+	blobs, err := d.Blobs()
+	if err != nil {
+		return err
+	}
+
+	for _, blob := range blobs {
+		blobID := blob.BlobstoreID
+		if blobID == "" {
+			blobID = "-"
+		}
+
+		if len(blob.HREF) > 0 {
+			digest, err := boshcrypto.ParseMultipleDigest(blob.SHA1)
+			if err != nil {
+				return bosherr.WrapErrorf(
+					err, "Generating multi digest for blob with path '%s' with digest string '%s'", blob.Path, blob.SHA1)
+			}
+
+			d.reporter.BlobDownloadStarted(blob.Path, blob.Size, blobID, blob.SHA1)
+
+			tempFile, err := d.downloadBlobHrefToTempFile(blob)
+			if err != nil {
+				d.reporter.BlobDownloadFinished(blob.Path, blobID, err)
+				return bosherr.WrapErrorf(err, "Downloading blob from '%s'", blob.HREF)
+			}
+
+			file, err := d.fs.OpenFile(tempFile.Name(), os.O_RDONLY, 0)
+			if err != nil {
+				d.reporter.BlobDownloadFinished(blob.Path, blobID, err)
+				return err
+			}
+			defer file.Close() //nolint:errcheck
+
+			if err := digest.Verify(file); err != nil {
+				d.reporter.BlobDownloadFinished(blob.Path, blobID, err)
+				return bosherr.WrapErrorf(err, "Validating SHA for '%s'", blob.HREF)
+			}
+
+			d.reporter.BlobDownloadFinished(blob.Path, blobID, nil)
+		} else {
+			d.reporter.BlobDownloadMessage(blob.Path, blobID, "skipped")
+		}
+	}
+	return nil
+}
+
 func (d FSBlobsDir) removeUnknownBlobs(blobs []Blob) error {
 	files, err := d.fs.RecursiveGlob(filepath.Join(d.dirPath, "**/*"))
 	if err != nil {
@@ -192,7 +243,7 @@ func (d FSBlobsDir) removeUnknownBlobs(blobs []Blob) error {
 	return nil
 }
 
-func (d FSBlobsDir) TrackBlob(path string, src io.ReadCloser) (Blob, error) {
+func (d FSBlobsDir) TrackBlob(path string, src io.ReadCloser, href string) (Blob, error) {
 	tempFile, err := d.fs.TempFile("track-blob")
 	if err != nil {
 		return Blob{}, bosherr.WrapErrorf(err, "Creating temp blob")
@@ -235,7 +286,7 @@ func (d FSBlobsDir) TrackBlob(path string, src io.ReadCloser) (Blob, error) {
 		idx = len(blobs) - 1
 	}
 
-	blobs[idx] = Blob{Path: path, Size: fileInfo.Size(), SHA1: sha1}
+	blobs[idx] = Blob{Path: path, Size: fileInfo.Size(), SHA1: sha1, HREF: href}
 
 	tempFile.Close() //nolint:errcheck
 
@@ -355,6 +406,64 @@ func (d FSBlobsDir) downloadBlob(blob Blob) error {
 	return d.moveBlobLocally(path, dstPath)
 }
 
+func (d FSBlobsDir) downloadBlobFromHref(blob Blob) error {
+	dstPath := filepath.Join(d.dirPath, blob.Path)
+
+	digest, err := boshcrypto.ParseMultipleDigest(blob.SHA1)
+	if err != nil {
+		return bosherr.WrapErrorf(
+			err, "Generating multi digest for blob with path '%s' with digest string '%s'", blob.Path, blob.SHA1)
+	}
+
+	if d.checkBlobExistence(dstPath, digest) {
+		return nil
+	}
+
+	blobID := blob.BlobstoreID
+	if blobID == "" {
+		blobID = "-"
+	}
+
+	d.reporter.BlobDownloadStarted(blob.Path, blob.Size, blobID, blob.SHA1)
+
+	tempFile, err := d.downloadBlobHrefToTempFile(blob)
+	if err != nil {
+		d.reporter.BlobDownloadFinished(blob.Path, "-", err)
+		return bosherr.WrapErrorf(err, "Downloading blob from '%s'", blob.HREF)
+	}
+
+	d.reporter.BlobDownloadFinished(blob.Path, blobID, nil)
+
+	return d.moveBlobLocally(tempFile.Name(), dstPath)
+}
+
+func (d FSBlobsDir) downloadBlobHrefToTempFile(blob Blob) (boshsys.File, error) {
+	resp, err := http.Get(blob.HREF)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close() //nolint:errcheck
+
+	tempFile, err := d.fs.TempFile("track-blob")
+	if err != nil {
+		return nil, err
+	}
+	defer tempFile.Close() //nolint:errcheck
+
+	_, err = io.Copy(tempFile, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := d.fs.OpenFile(tempFile.Name(), os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close() //nolint:errcheck
+
+	return tempFile, nil
+}
+
 func (d FSBlobsDir) uploadBlob(blob Blob) (string, error) {
 	var blobID string
 
@@ -395,6 +504,7 @@ func (d FSBlobsDir) save(blobs []Blob) error {
 			Size:        blob.Size,
 			BlobstoreID: blob.BlobstoreID,
 			SHA1:        blob.SHA1,
+			HREF:        blob.HREF,
 		}
 	}
 

releasedir/fs_blobs_dir_test.go

@@ -490,7 +490,7 @@ bad-sha-blob.tgz:
 	Describe("TrackBlob", func() {
 		act := func() (Blob, error) {
 			content := io.NopCloser(strings.NewReader("content"))
-			return blobsDir.TrackBlob(filepath.Join("dir", "file.tgz"), content)
+			return blobsDir.TrackBlob(filepath.Join("dir", "file.tgz"), content, "")
 		}
 
 		BeforeEach(func() {

releasedir/fs_release_dir.go

@@ -346,3 +346,7 @@ func (d FSReleaseDir) lastDevOrFinalVersion(name string) (*semver.Version, Relea
 		return nil, nil, nil
 	}
 }
+
+func (d FSReleaseDir) ValidateBlobsFromOrigin() error {
+	return d.blobsDir.ValidateBlobsFromOrigin()
+}

releasedir/interfaces.go

@@ -42,6 +42,9 @@ type ReleaseDir interface {
 
 	// FinalizeRelease adds the Release to the final list so that it's consumable by others.
 	FinalizeRelease(release boshrel.Release, force bool) error
+
+	// Validate blob SHA's that have a remote reference to confirm source matches
+	ValidateBlobsFromOrigin() error
 }
 
 //counterfeiter:generate . Config
@@ -76,8 +79,9 @@ type BlobsDir interface {
 
 	SyncBlobs(numOfParallelWorkers int) error
 	UploadBlobs() error
+	ValidateBlobsFromOrigin() error
 
-	TrackBlob(string, io.ReadCloser) (Blob, error)
+	TrackBlob(string, io.ReadCloser, string) (Blob, error)
 	UntrackBlob(string) error
 }
 
@@ -86,6 +90,7 @@ type BlobsDir interface {
 type BlobsDirReporter interface {
 	BlobDownloadStarted(path string, size int64, blobID, sha1 string)
 	BlobDownloadFinished(path, blobID string, err error)
+	BlobDownloadMessage(path, blobId, message string)
 
 	BlobUploadStarted(path string, size int64, sha1 string)
 	BlobUploadFinished(path, blobID string, err error)
@@ -97,6 +102,8 @@ type Blob struct {
 
 	BlobstoreID string
 	SHA1        string
+
+	HREF string
 }
 
 //counterfeiter:generate . ReleaseIndex