Skip to content

Commit 6bdc237

Browse files
sjjy123sjjy123
committed
[feat][meta] Modelscope download and access
1 parent 4a2e5d1 commit 6bdc237

12 files changed

Lines changed: 854 additions & 17 deletions

File tree

‎cmd/wire_gen.go‎

Lines changed: 3 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/config.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,12 @@ dynamicProxy:
7575
timePeriod: 60 #定期检测代理是否可用时间周期，单位秒（S）
7676
maxContinuousFails: 5 #连续失败次数超过该值，则认为代理不可用
7777
webhook: https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=73662ac1-1055-48a7-8c89-37964b5f4fdc111 # 企业微信机器人Webhook地址
78+
79+
modelscope:
80+
modelCacheRoot: ./repos/modelscope/models # 模型缓存根目录
81+
datasetCacheRoot: ./repos/modelscope/datasets # 数据集缓存根目录
82+
officialBaseURL: https://www.modelscope.cn # ModelScope官方基础地址
83+
chunkSize: 8388608 # 8MB分块，16*1024*1024的数值结果
84+
maxRetry: 5 # 超时重试次数
85+
retryDelay: 3 # 重试间隔，单位秒（S）（原配置为5*time.Second，YAML中简化为数值+注释）
86+
minFileSize: 1 # 最小缓存文件大小，单位字节（B）

‎internal/handler/handler.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,4 @@ import (
1818
"github.com/google/wire"
1919
)
2020

21-
var HandlerProvider = wire.NewSet(NewFileHandler, NewMetaHandler, NewSysHandler, NewCacheJobHandler)
21+
var HandlerProvider = wire.NewSet(NewFileHandler, NewMetaHandler, NewSysHandler, NewCacheJobHandler, NewModelscopeHandler)

‎internal/handler/modelscope_handler.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package handler
2+
3+
import (
4+
"strings"
5+
6+
"dingospeed/internal/service"
7+
"dingospeed/pkg/util"
8+
9+
"github.com/labstack/echo/v4"
10+
)
11+
12+
// ModelscopeHandler 模型代理请求处理器
13+
type ModelscopeHandler struct {
14+
ModelscopeService *service.ModelscopeService
15+
}
16+
17+
// NewModelscopeHandler 创建模型代理处理器实例
18+
func NewModelscopeHandler(ModelscopeService *service.ModelscopeService) *ModelscopeHandler {
19+
return &ModelscopeHandler{
20+
ModelscopeService: ModelscopeService,
21+
}
22+
}
23+
24+
// ModelInfoHandler 处理模型信息查询请求
25+
func (h *ModelscopeHandler) ModelInfoHandler(c echo.Context) error {
26+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
27+
28+
org, repo, repoType := parts[3], parts[4], parts[2]
29+
if err := h.ModelscopeService.ForwardModelInfo(c, org, repo, repoType); err != nil {
30+
return util.ResponseError(c, err)
31+
}
32+
return nil
33+
}
34+
35+
// RevisionsHandler 处理模型版本查询请求
36+
func (h *ModelscopeHandler) RevisionsHandler(c echo.Context) error {
37+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
38+
39+
org, repo, repoType := parts[3], parts[4], parts[2]
40+
if err := h.ModelscopeService.ForwardRevisions(c, org, repo, repoType); err != nil {
41+
return util.ResponseError(c, err)
42+
}
43+
return nil
44+
}
45+
46+
// FileListHandler 处理模型文件列表请求
47+
func (h *ModelscopeHandler) FileListHandler(c echo.Context) error {
48+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
49+
50+
org, repo, repoType := parts[3], parts[4], parts[2]
51+
if err := h.ModelscopeService.ForwardFileList(c, org, repo, repoType); err != nil {
52+
return util.ResponseError(c, err)
53+
}
54+
return nil
55+
}
56+
57+
// FileDownloadHandler 处理模型文件下载请求（支持续传）
58+
func (h *ModelscopeHandler) FileDownloadHandler(c echo.Context) error {
59+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
60+
61+
org, repo, repoType := parts[3], parts[4], parts[2]
62+
if err := h.ModelscopeService.HandleFileDownload(c, org, repo, repoType); err != nil {
63+
return util.ResponseError(c, err)
64+
}
65+
return nil
66+
}
67+
68+
// FileTreeHandler 处理数据集文件列表请求
69+
func (h *ModelscopeHandler) FileTreeHandler(c echo.Context) error {
70+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
71+
72+
org, repo, repoType := parts[3], parts[4], parts[2]
73+
if err := h.ModelscopeService.ForwardRepoTree(c, org, repo, repoType); err != nil {
74+
return util.ResponseError(c, err)
75+
}
76+
return nil
77+
}
78+
79+
// DatasetFileTreeHandler 处理数据集文件列表请求
80+
func (h *ModelscopeHandler) DatasetFileTreeHandler(c echo.Context) error {
81+
parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
82+
83+
datasetId := parts[3]
84+
if err := h.ModelscopeService.ForwardRepoTreeByDatasetId(c, datasetId); err != nil {
85+
return util.ResponseError(c, err)
86+
}
87+
return nil
88+
}

‎internal/router/http_router.go‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,23 @@ import (
2323
)
2424

2525
type HttpRouter struct {
26-
echo *echo.Echo
27-
fileHandler *handler.FileHandler
28-
metaHandler *handler.MetaHandler
29-
sysHandler *handler.SysHandler
30-
cacheJobHandler *handler.CacheJobHandler
26+
echo *echo.Echo
27+
fileHandler *handler.FileHandler
28+
metaHandler *handler.MetaHandler
29+
sysHandler *handler.SysHandler
30+
cacheJobHandler *handler.CacheJobHandler
31+
modelscopeHandler *handler.ModelscopeHandler
3132
}
3233

3334
func NewHttpRouter(echo *echo.Echo, fileHandler *handler.FileHandler, metaHandler *handler.MetaHandler,
34-
sysHandler *handler.SysHandler, cacheJobHandler *handler.CacheJobHandler) *HttpRouter {
35+
sysHandler *handler.SysHandler, cacheJobHandler *handler.CacheJobHandler, modelscopeHandler *handler.ModelscopeHandler) *HttpRouter {
3536
r := &HttpRouter{
36-
echo: echo,
37-
fileHandler: fileHandler,
38-
metaHandler: metaHandler,
39-
sysHandler: sysHandler,
40-
cacheJobHandler: cacheJobHandler,
37+
echo: echo,
38+
fileHandler: fileHandler,
39+
metaHandler: metaHandler,
40+
sysHandler: sysHandler,
41+
cacheJobHandler: cacheJobHandler,
42+
modelscopeHandler: modelscopeHandler,
4143
}
4244
r.initRouter()
4345
return r
@@ -54,6 +56,7 @@ func (r *HttpRouter) initRouter() {
5456
r.routerForCacheJob()
5557

5658
r.routerForSpeed()
59+
r.routerForModelscope()
5760
}
5861

5962
func (r *HttpRouter) routerForSpeed() { // alayanew
@@ -91,3 +94,12 @@ func (r *HttpRouter) routerForCacheJob() { // alayanew
9194
r.echo.POST("/api/cacheJob/resume", r.cacheJobHandler.ResumeCacheJobHandler)
9295
r.echo.POST("/api/cacheJob/realtime", r.cacheJobHandler.RealtimeCacheJobHandler)
9396
}
97+
98+
func (r *HttpRouter) routerForModelscope() { // modelscope
99+
r.echo.GET("/api/v1/:repoType/:org/:repo", r.modelscopeHandler.ModelInfoHandler)
100+
r.echo.GET("/api/v1/:repoType/:org/:repo/revisions", r.modelscopeHandler.RevisionsHandler)
101+
r.echo.GET("/api/v1/:repoType/:org/:repo/repo/files", r.modelscopeHandler.FileListHandler)
102+
r.echo.GET("/api/v1/:repoType/:org/:repo/repo", r.modelscopeHandler.FileDownloadHandler)
103+
r.echo.GET("/api/v1/:repoType/:org/:repo/repo/tree", r.modelscopeHandler.FileTreeHandler)
104+
r.echo.GET("/api/v1/datasets/:datasetId/repo/tree", r.modelscopeHandler.DatasetFileTreeHandler)
105+
}

‎internal/server/http.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ func NewEngine() *echo.Echo {
8080
r := echo.New()
8181
middleware.InitMiddlewareConfig()
8282
r.Use(middleware.QueueLimitMiddleware)
83+
r.Use(middleware.CORSMiddleware())
8384

8485
t := &Template{
8586
templates: template.Must(template.ParseFS(templatesFS, "templates/*.html")),

0 commit comments

Comments
 (0)