Add upload and git import skills (#957)

* Add upload and git import skills

* Fix lint

* Fix test

* Fix lint

---------

Co-authored-by: Dev Agent <dev-agent@example.com>
Co-authored-by: Zhang ZeHua <zh.zhang@opencsg.com>

Author: 3 people

_mocks/opencsg.com/csghub-server/builder/store/s3/mock_Client.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"opencsg.com/csghub-server/api/httpbase"
+	"opencsg.com/csghub-server/builder/store/s3"
 	"opencsg.com/csghub-server/common/config"
 	"opencsg.com/csghub-server/common/errorx"
 	"opencsg.com/csghub-server/common/types"
@@ -29,17 +30,25 @@ func NewSkillHandler(config *config.Config) (*SkillHandler, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error creating repo component:%w", err)
 	}
+	s3Client, err := s3.NewMinio(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating s3 client:%w", err)
+	}
 	return &SkillHandler{
 		skill:     tc,
 		sensitive: sc,
 		repo:      repo,
+		s3Client:  s3Client,
+		config:    config,
 	}, nil
 }
 
 type SkillHandler struct {
 	skill     component.SkillComponent
 	sensitive component.SensitiveComponent
 	repo      component.RepoComponent
+	s3Client  s3.Client
+	config    *config.Config
 }
 
 // CreateSkill   godoc
@@ -338,3 +347,31 @@ func (h *SkillHandler) Relations(ctx *gin.Context) {
 
 	httpbase.OK(ctx, detail)
 }
+
+// UploadSkillPackage godoc
+// @Security     ApiKey
+// @Summary      Get skill package upload URL
+// @Description  Get a presigned URL and form data for uploading skill package
+// @Tags         Skill
+// @Produce      json
+// @Param        current_user query string false "current user"
+// @Success      200  {object}  types.Response{data=map[string]interface{}} "OK"
+// @Failure      400  {object}  types.APIBadRequest "Bad request"
+// @Failure      500  {object}  types.APIInternalServerError "Internal server error"
+// @Router       /skills/upload_url [post]
+func (h *SkillHandler) GetUploadUrl(ctx *gin.Context) {
+	// Call component to get upload URL, UUID, and form data
+	url, uuid, formData, err := h.skill.GetUploadUrl(ctx.Request.Context())
+	if err != nil {
+		slog.ErrorContext(ctx.Request.Context(), "Failed to get upload URL", slog.Any("error", err))
+		httpbase.ServerError(ctx, err)
+		return
+	}
+
+	// Return the upload URL, UUID, and form data
+	httpbase.OK(ctx, map[string]interface{}{
+		"url":      url,
+		"uuid":     uuid,
+		"formData": formData,
+	})
+}

_mocks/opencsg.com/csghub-server/component/mock_SkillComponent.go

@@ -1,14 +1,19 @@
 package handler
 
 import (
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"github.com/alibabacloud-go/tea/tea"
 	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	mockcomponent "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component"
 	"opencsg.com/csghub-server/builder/testutil"
+	"opencsg.com/csghub-server/common/config"
 	"opencsg.com/csghub-server/common/types"
 )
 
@@ -169,3 +174,74 @@ func TestSkillHandler_Relations(t *testing.T) {
 	tester.WithUser().Execute()
 	tester.ResponseEq(t, 200, tester.OKText, &types.Relations{})
 }
+
+func TestSkillHandler_GetUploadUrl(t *testing.T) {
+	// Setup test configuration
+	cfg := &config.Config{}
+	cfg.S3.Bucket = "test-bucket"
+
+	// Create mock components
+	mockSkillComponent := mockcomponent.NewMockSkillComponent(t)
+	mockSensitiveComponent := mockcomponent.NewMockSensitiveComponent(t)
+	mockRepoComponent := mockcomponent.NewMockRepoComponent(t)
+
+	// Expected upload URL, UUID, and form data
+	expectedURL := "http://example.com/upload"
+	expectedUUID := "test-uuid"
+	expectedFormData := map[string]string{
+		"key":             "skills/packages/test-uuid",
+		"policy":          "test-policy",
+		"x-amz-signature": "test-signature",
+	}
+
+	// Set up mock expectations
+	mockSkillComponent.EXPECT().GetUploadUrl(mock.Anything).Return(expectedURL, expectedUUID, expectedFormData, nil)
+
+	// Create skill handler with mock dependencies
+	handler := &SkillHandler{
+		skill:     mockSkillComponent,
+		sensitive: mockSensitiveComponent,
+		repo:      mockRepoComponent,
+		config:    cfg,
+	}
+
+	// Create a test HTTP request
+	req, err := http.NewRequest("POST", "/skills/upload_url", nil)
+	require.Nil(t, err)
+	// Set the current user header
+	req.Header.Set("X-User", "test-user")
+
+	// Create a test HTTP response recorder
+	w := httptest.NewRecorder()
+
+	// Create a gin context
+	ctx, _ := gin.CreateTestContext(w)
+	ctx.Request = req
+	// Set current user in context
+	ctx.Set("currentUser", "test-user")
+
+	// Call the GetUploadUrl method
+	handler.GetUploadUrl(ctx)
+
+	// Check the response
+	require.Equal(t, http.StatusOK, w.Code)
+
+	// Check the response body
+	var response struct {
+		Msg  string                 `json:"msg"`
+		Data map[string]interface{} `json:"data"`
+	}
+	err = json.Unmarshal(w.Body.Bytes(), &response)
+	require.Nil(t, err)
+	require.Equal(t, expectedURL, response.Data["url"])
+	require.Equal(t, expectedUUID, response.Data["uuid"])
+
+	// Convert formData to map[string]string for comparison
+	formData, ok := response.Data["formData"].(map[string]interface{})
+	require.True(t, ok)
+	expectedFormDataMap := make(map[string]interface{})
+	for k, v := range expectedFormData {
+		expectedFormDataMap[k] = v
+	}
+	require.Equal(t, expectedFormDataMap, formData)
+}

api/handler/skill.go

@@ -1462,6 +1462,11 @@ func createSkillRoutes(
 	repoCommonHandler *handler.RepoHandler,
 ) {
 	skillGroup := apiGroup.Group("/skills")
+
+	// Upload skill package endpoint - doesn't require existing repo
+	skillGroup.POST("/upload_url", middlewareCollection.Auth.NeedPhoneVerified, skillHandler.GetUploadUrl)
+
+	// Routes that require existing repo
 	skillGroup.Use(middleware.RepoType(types.SkillRepo), middlewareCollection.Repo.RepoExists)
 	{
 		skillGroup.POST("", middlewareCollection.Auth.NeedPhoneVerified, skillHandler.Create)

api/handler/skill_test.go

@@ -72,6 +72,7 @@ type MinioClient interface {
 	CopyObject(ctx context.Context, dst minio.CopyDestOptions, src minio.CopySrcOptions) (minio.UploadInfo, error)
 	GetObject(ctx context.Context, bucketName, objectName string, opts minio.GetObjectOptions) (*minio.Object, error)
 	ListObjects(ctx context.Context, bucketName string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo
+	PresignedPostPolicy(ctx context.Context, policy *minio.PostPolicy) (u *url.URL, formData map[string]string, err error)
 }
 
 type Client interface {

api/router/api.go

@@ -4,6 +4,14 @@ import "time"
 
 type CreateSkillReq struct {
 	CreateRepoReq
+	// Skill package SHA256 hash
+	SkillPackageSHA256 string `json:"skill_file"`
+	// Git repository URL for mirroring
+	GitURL string `json:"git_url"`
+	// Git username for authentication
+	GitUsername string `json:"git_username"`
+	// Git password for authentication
+	GitPassword string `json:"git_password"`
 }
 
 type UpdateSkillReq struct {

builder/store/s3/minio.go

@@ -256,3 +256,8 @@ func ShortenCommitID7(fullCommitID string) (string, error) {
 	}
 	return commitID[:7], nil
 }
+
+// BuildSkillPackageObjectKey builds the object key for skill package in object storage
+func BuildSkillPackageObjectKey(sha256 string) string {
+	return fmt.Sprintf("skills/packages/%s", sha256)
+}

common/types/skill.go

@@ -373,3 +373,17 @@ func TestBuildHashedRelativePath(t *testing.T) {
 	res := BuildHashedRelativePath(1)
 	require.Equal(t, "@hashed_repos/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b.git", res)
 }
+
+func TestBuildSkillPackageObjectKey(t *testing.T) {
+	// Test with a sample SHA256 hash
+	sha256 := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	expectedObjectKey := "skills/packages/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	actualObjectKey := BuildSkillPackageObjectKey(sha256)
+	require.Equal(t, expectedObjectKey, actualObjectKey)
+
+	// Test with another SHA256 hash
+	sha256 = "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8"
+	expectedObjectKey = "skills/packages/5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8"
+	actualObjectKey = BuildSkillPackageObjectKey(sha256)
+	require.Equal(t, expectedObjectKey, actualObjectKey)
+}