Merge pull request #1550 from linuxdataflow/feat/pls/gcs-support

Add support for GCS.

Author: neolynx

.golangci.yml

@@ -1,4 +1,6 @@
 version: "2"
+run:
+  timeout: 5m
 linters:
   settings:
     staticcheck:

api/api_test.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"sort"
 	"strings"
 	"testing"
 
@@ -41,6 +42,17 @@ func createTestConfig() *os.File {
 	jsonString, err := json.Marshal(gin.H{
 		"architectures":         []string{},
 		"enableMetricsEndpoint": true,
+		"S3PublishEndpoints": map[string]map[string]string{
+			"test-s3": {
+				"region": "us-east-1",
+				"bucket": "bucket-s3",
+			},
+		},
+		"GcsPublishEndpoints": map[string]map[string]string{
+			"test-gcs": {
+				"bucket": "bucket-gcs",
+			},
+		},
 	})
 	if err != nil {
 		return nil
@@ -173,3 +185,27 @@ func (s *APISuite) TestTruthy(c *C) {
 	c.Check(truthy(-1), Equals, true)
 	c.Check(truthy(gin.H{}), Equals, true)
 }
+
+func (s *APISuite) TestGetS3Endpoints(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/s3", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+
+	var endpoints []string
+	err = json.Unmarshal(response.Body.Bytes(), &endpoints)
+	c.Assert(err, IsNil)
+	sort.Strings(endpoints)
+	c.Check(endpoints, DeepEquals, []string{"test-s3"})
+}
+
+func (s *APISuite) TestGetGCSEndpoints(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/gcs", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+
+	var endpoints []string
+	err = json.Unmarshal(response.Body.Bytes(), &endpoints)
+	c.Assert(err, IsNil)
+	sort.Strings(endpoints)
+	c.Check(endpoints, DeepEquals, []string{"test-gcs"})
+}

api/gcs.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary GCS buckets
+// @Description **Get list of GCS buckets**
+// @Description
+// @Description List configured GCS buckets.
+// @Tags Status
+// @Produce json
+// @Success 200 {array} string "List of GCS buckets"
+// @Router /api/gcs [get]
+func apiGCSList(c *gin.Context) {
+	keys := []string{}
+	for k := range context.Config().GCSPublishRoots {
+		keys = append(keys, k)
+	}
+	c.JSON(200, keys)
+}

api/router.go

@@ -177,6 +177,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 	{
 		api.GET("/s3", apiS3List)
+		api.GET("/gcs", apiGCSList)
 	}
 
 	{

context/context.go

@@ -23,6 +23,7 @@ import (
 	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/gcs"
 	"github.com/aptly-dev/aptly/http"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/s3"
@@ -437,6 +438,20 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 			if err != nil {
 				Fatal(err)
 			}
+		} else if strings.HasPrefix(name, "gcs:") {
+			params, ok := context.config().GCSPublishRoots[name[4:]]
+			if !ok {
+				Fatal(fmt.Errorf("published GCS storage %v not configured", name[4:]))
+			}
+
+			var err error
+			publishedStorage, err = gcs.NewPublishedStorage(
+				params.Bucket, params.Prefix, params.CredentialsFile, params.ServiceAccountJSON,
+				params.Project, params.Endpoint, params.ACL, params.StorageClass, params.EncryptionKey,
+				params.DisableMultiDel, params.Debug)
+			if err != nil {
+				Fatal(err)
+			}
 		} else if strings.HasPrefix(name, "swift:") {
 			params, ok := context.config().SwiftPublishRoots[name[6:]]
 			if !ok {

debian/aptly.conf

@@ -256,6 +256,58 @@ s3_publish_endpoints:
     #     # Enables detailed request/response dump for each S3 operation
     #     debug: false
 
+# GCS Endpoint Support
+#
+# aptly can be configured to publish repositories directly to Google Cloud
+# Storage. First, publishing endpoints should be described in the aptly
+# configuration file. Each endpoint has a name and associated settings.
+#
+# In order to publish to GCS, specify endpoint as `gcs:endpoint-name:` before
+# publishing prefix on the command line, e.g.:
+#
+#   `aptly publish snapshot wheezy-main gcs:test:`
+#
+gcs_publish_endpoints:
+    # # Endpoint Name
+    # test:
+    #     # Bucket name
+    #     bucket: test-bucket
+    #     # Prefix (optional)
+    #     # publishing under specified prefix in the bucket, defaults to
+    #     # no prefix (bucket root)
+    #     prefix: ""
+    #     # Credentials File (optional)
+    #     # Path to a service account credentials JSON file
+    #     credentials_file: ""
+    #     # Service Account JSON (optional)
+    #     # Inline service account credentials JSON payload
+    #     service_account_json: ""
+    #     # Project (optional)
+    #     # Quota project used for GCS requests
+    #     project: ""
+    #     # Endpoint (optional)
+    #     # Override the GCS endpoint (e.g. for staging or a fake server);
+    #     # leave empty to use the default GCS endpoint
+    #     endpoint: ""
+    #     # Default ACLs (optional)
+    #     # assign ACL to published files:
+    #     # * private     (default)
+    #     # * public-read (public repository)
+    #     # * none        (don't set ACL)
+    #     acl: private
+    #     # Storage Class (optional)
+    #     # GCS storage class, e.g. `STANDARD`
+    #     storage_class: STANDARD
+    #     # Encryption Key (optional)
+    #     # Customer-supplied encryption key (32-byte AES-256 key)
+    #     encryption_key: ""
+    #     # Disable MultiDel (optional)
+    #     # Kept for parity with S3 settings; GCS deletes are one-by-one
+    #     disable_multidel: false
+    #     # Debug (optional)
+    #     # Enables detailed logs for each GCS operation
+    #     debug: false
+
 # Swift Endpoint Support
 #
 # aptly can publish a repository directly to OpenStack Swift.

gcs/gcs.go

@@ -0,0 +1,2 @@
+// Package gcs handles publishing to Google Cloud Storage.
+package gcs

gcs/gcs_test.go

@@ -0,0 +1,12 @@
+package gcs
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests.
+func Test(t *testing.T) {
+	TestingT(t)
+}