Skip to content

Commit 7e71a70

Browse files
sjjy123sjjy123
committed
[feat][meta] Modelscope download and access
1 parent 8d8c7df commit 7e71a70

13 files changed

Lines changed: 949 additions & 17 deletions

File tree

‎cmd/wire_gen.go‎

Lines changed: 3 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/config.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,9 @@ dynamicProxy:
7575
timePeriod: 60 #定期检测代理是否可用时间周期，单位秒（S）
7676
maxContinuousFails: 5 #连续失败次数超过该值，则认为代理不可用
7777
webhook: https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=73662ac1-1055-48a7-8c89-37964b5f4fdc111 # 企业微信机器人Webhook地址
78+
79+
modelscope:
80+
officialBaseURL: https://www.modelscope.cn # ModelScope官方基础地址
81+
chunkSize: 8388608 # 8MB分块，16*1024*1024的数值结果
82+
maxRetry: 5 # 超时重试次数
83+
retryDelay: 3 # 重试间隔，单位秒（S）（原配置为5*time.Second，YAML中简化为数值+注释）

‎internal/handler/handler.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,4 @@ import (
1818
"github.com/google/wire"
1919
)
2020

21-
var HandlerProvider = wire.NewSet(NewFileHandler, NewMetaHandler, NewSysHandler, NewCacheJobHandler)
21+
var HandlerProvider = wire.NewSet(NewFileHandler, NewMetaHandler, NewSysHandler, NewCacheJobHandler, NewModelscopeHandler)

‎internal/handler/modelscope_handler.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package handler
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"dingospeed/internal/service"
8+
"dingospeed/pkg/util"
9+
10+
"github.com/labstack/echo/v4"
11+
)
12+
13+
// ModelscopeHandler 模型代理请求处理器
14+
type ModelscopeHandler struct {
15+
modelscopeService *service.ModelscopeService
16+
}
17+
18+
// NewModelscopeHandler 创建模型代理处理器实例
19+
func NewModelscopeHandler(modelscopeService *service.ModelscopeService) *ModelscopeHandler {
20+
return &ModelscopeHandler{
21+
modelscopeService: modelscopeService,
22+
}
23+
}
24+
25+
// ModelInfoHandler 处理模型信息查询请求
26+
func (m *ModelscopeHandler) ModelInfoHandler(c echo.Context) error {
27+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
28+
29+
if len(parts) < 5 {
30+
err := fmt.Errorf("请求路径格式非法")
31+
return util.ResponseError(c, err)
32+
}
33+
34+
org, repo, repoType := parts[3], parts[4], parts[2]
35+
if err := m.modelscopeService.ForwardModelInfo(c, org, repo, repoType); err != nil {
36+
return util.ResponseError(c, err)
37+
}
38+
return nil
39+
}
40+
41+
// RevisionsHandler 处理模型版本查询请求
42+
func (m *ModelscopeHandler) RevisionsHandler(c echo.Context) error {
43+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
44+
45+
if len(parts) < 5 {
46+
err := fmt.Errorf("请求路径格式非法")
47+
return util.ResponseError(c, err)
48+
}
49+
50+
org, repo, repoType := parts[3], parts[4], parts[2]
51+
if err := m.modelscopeService.ForwardRevisions(c, org, repo, repoType); err != nil {
52+
return util.ResponseError(c, err)
53+
}
54+
return nil
55+
}
56+
57+
// FileListHandler 处理模型文件列表请求
58+
func (m *ModelscopeHandler) FileListHandler(c echo.Context) error {
59+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
60+
61+
if len(parts) < 5 {
62+
err := fmt.Errorf("请求路径格式非法")
63+
return util.ResponseError(c, err)
64+
}
65+
66+
org, repo, repoType := parts[3], parts[4], parts[2]
67+
if err := m.modelscopeService.ForwardFileList(c, org, repo, repoType); err != nil {
68+
return util.ResponseError(c, err)
69+
}
70+
return nil
71+
}
72+
73+
// FileDownloadHandler 处理模型文件下载请求（支持续传）
74+
func (m *ModelscopeHandler) FileDownloadHandler(c echo.Context) error {
75+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
76+
77+
if len(parts) < 5 {
78+
err := fmt.Errorf("请求路径格式非法")
79+
return util.ResponseError(c, err)
80+
}
81+
82+
org, repo, repoType := parts[3], parts[4], parts[2]
83+
if err := m.modelscopeService.HandleFileDownload(c, org, repo, repoType); err != nil {
84+
return util.ResponseError(c, err)
85+
}
86+
return nil
87+
}
88+
89+
// FileTreeHandler 处理数据集文件列表请求
90+
func (m *ModelscopeHandler) FileTreeHandler(c echo.Context) error {
91+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
92+
93+
if len(parts) < 5 {
94+
err := fmt.Errorf("请求路径格式非法")
95+
return util.ResponseError(c, err)
96+
}
97+
98+
org, repo, repoType := parts[3], parts[4], parts[2]
99+
if err := m.modelscopeService.ForwardRepoTree(c, org, repo, repoType); err != nil {
100+
return util.ResponseError(c, err)
101+
}
102+
return nil
103+
}
104+
105+
// DatasetFileTreeHandler 处理数据集文件列表请求
106+
func (m *ModelscopeHandler) DatasetFileTreeHandler(c echo.Context) error {
107+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
108+
109+
if len(parts) < 4 {
110+
err := fmt.Errorf("请求路径格式非法")
111+
return util.ResponseError(c, err)
112+
}
113+
114+
datasetId := parts[3]
115+
if err := m.modelscopeService.ForwardRepoTreeByDatasetId(c, datasetId); err != nil {
116+
return util.ResponseError(c, err)
117+
}
118+
return nil
119+
}

‎internal/router/http_router.go‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,23 @@ import (
2323
)
2424

2525
type HttpRouter struct {
26-
echo *echo.Echo
27-
fileHandler *handler.FileHandler
28-
metaHandler *handler.MetaHandler
29-
sysHandler *handler.SysHandler
30-
cacheJobHandler *handler.CacheJobHandler
26+
echo *echo.Echo
27+
fileHandler *handler.FileHandler
28+
metaHandler *handler.MetaHandler
29+
sysHandler *handler.SysHandler
30+
cacheJobHandler *handler.CacheJobHandler
31+
modelscopeHandler *handler.ModelscopeHandler
3132
}
3233

3334
func NewHttpRouter(echo *echo.Echo, fileHandler *handler.FileHandler, metaHandler *handler.MetaHandler,
34-
sysHandler *handler.SysHandler, cacheJobHandler *handler.CacheJobHandler) *HttpRouter {
35+
sysHandler *handler.SysHandler, cacheJobHandler *handler.CacheJobHandler, modelscopeHandler *handler.ModelscopeHandler) *HttpRouter {
3536
r := &HttpRouter{
36-
echo: echo,
37-
fileHandler: fileHandler,
38-
metaHandler: metaHandler,
39-
sysHandler: sysHandler,
40-
cacheJobHandler: cacheJobHandler,
37+
echo: echo,
38+
fileHandler: fileHandler,
39+
metaHandler: metaHandler,
40+
sysHandler: sysHandler,
41+
cacheJobHandler: cacheJobHandler,
42+
modelscopeHandler: modelscopeHandler,
4143
}
4244
r.initRouter()
4345
return r
@@ -54,6 +56,7 @@ func (r *HttpRouter) initRouter() {
5456
r.routerForCacheJob()
5557

5658
r.routerForSpeed()
59+
r.routerForModelscope()
5760
}
5861

5962
func (r *HttpRouter) routerForSpeed() { // alayanew
@@ -91,3 +94,12 @@ func (r *HttpRouter) routerForCacheJob() { // alayanew
9194
r.echo.POST("/api/cacheJob/resume", r.cacheJobHandler.ResumeCacheJobHandler)
9295
r.echo.POST("/api/cacheJob/realtime", r.cacheJobHandler.RealtimeCacheJobHandler)
9396
}
97+
98+
func (r *HttpRouter) routerForModelscope() { // modelscope
99+
r.echo.GET("/api/v1/:repoType/:org/:repo", r.modelscopeHandler.ModelInfoHandler)
100+
r.echo.GET("/api/v1/:repoType/:org/:repo/revisions", r.modelscopeHandler.RevisionsHandler)
101+
r.echo.GET("/api/v1/:repoType/:org/:repo/repo/files", r.modelscopeHandler.FileListHandler)
102+
r.echo.GET("/api/v1/:repoType/:org/:repo/repo", r.modelscopeHandler.FileDownloadHandler)
103+
r.echo.GET("/api/v1/:repoType/:org/:repo/repo/tree", r.modelscopeHandler.FileTreeHandler)
104+
r.echo.GET("/api/v1/datasets/:datasetId/repo/tree", r.modelscopeHandler.DatasetFileTreeHandler)
105+
}

‎internal/server/http.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ func NewEngine() *echo.Echo {
8080
r := echo.New()
8181
middleware.InitMiddlewareConfig()
8282
r.Use(middleware.QueueLimitMiddleware)
83+
r.Use(middleware.CORSMiddleware())
8384

8485
t := &Template{
8586
templates: template.Must(template.ParseFS(templatesFS, "templates/*.html")),

0 commit comments

Comments
 (0)