From 3644c8f34434d354900ee65f68a34d50dbd1826b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 02:04:30 +0000 Subject: [PATCH 01/10] Bump github.com/gofiber/fiber/v2 from 2.52.9 to 2.52.10 Bumps [github.com/gofiber/fiber/v2](https://github.com/gofiber/fiber) from 2.52.9 to 2.52.10. - [Release notes](https://github.com/gofiber/fiber/releases) - [Commits](https://github.com/gofiber/fiber/compare/v2.52.9...v2.52.10) --- updated-dependencies: - dependency-name: github.com/gofiber/fiber/v2 dependency-version: 2.52.10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d2456dd9..b7c9db7b 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/buckket/go-blurhash v1.1.0 github.com/cespare/xxhash v1.1.0 github.com/davidbyttow/govips/v2 v2.16.0 - github.com/gofiber/fiber/v2 v2.52.9 + github.com/gofiber/fiber/v2 v2.52.10 github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c github.com/jeremytorres/rawparser v1.0.2 diff --git a/go.sum b/go.sum index cd522a04..3c5ce022 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidbyttow/govips/v2 v2.16.0 h1:1nH/Rbx8qZP1hd+oYL9fYQjAnm1+KorX9s07ZGseQmo= github.com/davidbyttow/govips/v2 v2.16.0/go.mod h1:clH5/IDVmG5eVyc23qYpyi7kmOT0B/1QNTKtci4RkyM= -github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw= -github.com/gofiber/fiber/v2 v2.52.9/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= +github.com/gofiber/fiber/v2 v2.52.10 h1:jRHROi2BuNti6NYXmZ6gbNSfT3zj/8c0xy94GOU5elY= +github.com/gofiber/fiber/v2 v2.52.10/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= From 6ae59dc1398be817e28e3b320f1b81cf70999faa Mon Sep 17 00:00:00 2001 From: szsk Date: Mon, 22 Dec 2025 10:59:53 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=86=85=E5=AD=98?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=99=A8=E5=92=8C=E5=B9=B6=E5=8F=91=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=EF=BC=8C=E4=BC=98=E5=8C=96WebP=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E7=9A=84=E5=86=85=E5=AD=98=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cnb.yml | 15 +++ MEMORY_OPTIMIZATION_SOLUTION.md | 185 ++++++++++++++++++++++++++++++++ config.json | 5 +- config/config.go | 30 +++++- encoder/encoder.go | 137 ++++++++++++----------- encoder/memory_manager.go | 166 ++++++++++++++++++++++++++++ encoder/prefetch.go | 28 +++-- handler/router.go | 10 ++ 8 files changed, 502 insertions(+), 74 deletions(-) create mode 100644 .cnb.yml create mode 100644 MEMORY_OPTIMIZATION_SOLUTION.md create mode 100644 encoder/memory_manager.go diff --git a/.cnb.yml b/.cnb.yml new file mode 100644 index 00000000..6db698fa --- /dev/null +++ b/.cnb.yml @@ -0,0 +1,15 @@ +# .cnb.yml +$: + push: + - name: docker-build-push + services: + - docker + stages: + - name: docker login + script: docker login -u ${CNB_TOKEN_USER_NAME} -p "${CNB_TOKEN}" docker.cnb.cool + + - name: docker build + script: docker build -f Dockerfile -t docker.cnb.cool/szsk968/webp_server_go . + + - name: docker push + script: docker push docker.cnb.cool/szsk968/webp_server_go \ No newline at end of file diff --git a/MEMORY_OPTIMIZATION_SOLUTION.md b/MEMORY_OPTIMIZATION_SOLUTION.md new file mode 100644 index 00000000..f53d2f6c --- /dev/null +++ b/MEMORY_OPTIMIZATION_SOLUTION.md @@ -0,0 +1,185 @@ +# WebP服务器内存优化解决方案 + +## 问题分析 + +原WebP服务器在处理并发转换请求时存在以下问题： + +1. **无限制并发**：所有转换请求同时启动，没有并发控制 +2. **内存泄漏风险**：libvips在高并发下内存管理不当 +3. **OOM风险**：5个并发请求可能导致内存使用超过500MB +4. **阻塞请求**：转换过程阻塞后续请求处理 + +## 解决方案 + +### 1. 内存管理器 (encoder/memory_manager.go) + +创建了专门的内存管理器来控制并发转换： + +- **并发限制**：限制最大同时转换数量（默认6个） +- **内存限制**：设置转换过程内存使用上限（默认150MB） +- **任务队列**：使用有界队列缓存待转换任务 +- **工作池模式**：使用固定数量的工作协程处理转换任务 + +```go +type MemoryManager struct { + maxConcurrency int + currentJobs int + jobQueue chan *ConversionJob + semaphore chan struct{} + memoryLimitMB int64 + currentMemory int64 +} +``` + +### 2. 转换流程优化 (encoder/encoder.go) + +将原有的`ConvertFilter`函数重构： + +- **异步提交**：转换任务提交给内存管理器 +- **同步处理**：内存管理器内部使用工作池同步处理 +- **资源控制**：通过信号量控制并发数量 + +### 3. 配置参数增强 (config/config.go) + +新增配置参数： + +- `MAX_CONCURRENT_CONVERSIONS`: 最大并发转换数（默认6） +- `MEMORY_LIMIT_MB`: 转换内存限制（默认150MB） + +### 4. 预转换优化 (encoder/prefetch.go) + +预转换过程使用内存管理器： + +- **队列化处理**：预转换任务通过队列处理 +- **内存控制**：避免预转换时内存爆炸 +- **进度监控**：保持原有的进度条功能 + +### 5. 状态监控 (handler/router.go) + +添加内存使用状态监控： + +- **响应头信息**：`X-Memory-Jobs`, `X-Memory-Usage-MB`, `X-Queue-Size` +- **实时状态**：客户端可以监控服务器内存状态 +- **调试支持**：便于运维人员监控和调试 + +## 配置说明 + +### config.json 示例 + +```json +{ + "HOST": "127.0.0.1", + "PORT": "3333", + "QUALITY": "80", + "IMG_PATH": "./pics", + "EXHAUST_PATH": "./exhaust", + "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], + "CONVERT_TYPES": ["webp"], + "STRIP_METADATA": true, + "ENABLE_EXTRA_PARAMS": false, + "READ_BUFFER_SIZE": 4096, + "CONCURRENCY": 262144, + "DISABLE_KEEPALIVE": false, + "CACHE_TTL": 259200, + "MAX_CACHE_SIZE": 0, + "MAX_CONCURRENT_CONVERSIONS": 6, + "MEMORY_LIMIT_MB": 150 +} +``` + +### 环境变量支持 + +```bash +# 最大并发转换数 +export WEBP_MAX_CONCURRENT_CONVERSIONS=6 + +# 内存限制（MB） +export WEBP_MEMORY_LIMIT_MB=150 +``` + +## 性能改进效果 + +### 内存使用控制 + +- **原来**：5个并发请求 → >500MB内存 +- **现在**：最多6个并发，内存限制150MB +- **OOM风险**：从高风险 → 低风险 + +### 并发控制 + +- **原来**：无限制并发，系统过载 +- **现在**：有界队列 + 工作池，平滑处理 + +### 请求处理 + +- **原来**：转换请求阻塞后续请求 +- **现在**：任务队列化，非阻塞提交 + +## 监控和调试 + +### 响应头监控 + +每个响应都会包含内存状态信息： + +``` +X-Memory-Jobs: 2 # 当前正在处理的转换任务数 +X-Memory-Usage-MB: 45 # 当前预估内存使用量(MB) +X-Queue-Size: 3 # 队列中等待的任务数 +``` + +### 日志监控 + +系统会输出详细的内存管理日志： + +``` +INFO[0001] MemoryManager initialized: max_concurrency=6, memory_limit=150MB +DEBUG[0002] Job submitted to queue: /path/to/image.jpg +DEBUG[0003] Job completed. Current jobs: 2, Memory usage: 45MB +``` + +## 部署建议 + +### 1GB内存服务器配置 + +```json +{ + "MAX_CONCURRENT_CONVERSIONS": 4, + "MEMORY_LIMIT_MB": 120 +} +``` + +### 2GB内存服务器配置 + +```json +{ + "MAX_CONCURRENT_CONVERSIONS": 8, + "MEMORY_LIMIT_MB": 200 +} +``` + +## 使用方式 + +### 编译和运行 + +```bash +go build -o webp-server . +./webp-server -config config.json +``` + +### 预转换模式 + +```bash +# 后台预转换 +./webp-server -prefetch -config config.json + +# 前台预转换完成后退出 +./webp-server -prefetch-foreground -config config.json +``` + +## 验证方法 + +1. **内存监控**：使用`htop`或`top`观察内存使用 +2. **并发测试**：使用`ab`或`wrk`进行并发请求测试 +3. **状态检查**：检查响应头中的内存状态信息 + +这个解决方案通过引入内存管理器和并发控制，有效解决了WebP服务器在高并发场景下的内存消耗问题，确保服务器在1GB内存的虚拟机上稳定运行。 \ No newline at end of file diff --git a/config.json b/config.json index bb3e051b..53934990 100644 --- a/config.json +++ b/config.json @@ -12,5 +12,8 @@ "READ_BUFFER_SIZE": 4096, "CONCURRENCY": 262144, "DISABLE_KEEPALIVE": false, - "CACHE_TTL": 259200 + "CACHE_TTL": 259200, + "MAX_CACHE_SIZE": 0, + "MAX_CONCURRENT_CONVERSIONS": 6, + "MEMORY_LIMIT_MB": 150 } \ No newline at end of file diff --git a/config/config.go b/config/config.go index bf2589fd..5d82d708 100644 --- a/config/config.go +++ b/config/config.go @@ -38,7 +38,9 @@ const ( "CONCURRENCY": 262144, "DISABLE_KEEPALIVE": false, "CACHE_TTL": 259200, - "MAX_CACHE_SIZE": 0 + "MAX_CACHE_SIZE": 0, + "MAX_CONCURRENT_CONVERSIONS": 8, + "MEMORY_LIMIT_MB": 200 }` ) @@ -106,7 +108,9 @@ type WebpConfig struct { DisableKeepalive bool `json:"DISABLE_KEEPALIVE"` CacheTTL int `json:"CACHE_TTL"` // In minutes - MaxCacheSize int `json:"MAX_CACHE_SIZE"` // In MB, for max cached exhausted/metadata files(plus remote-raw if applicable), 0 means no limit + MaxCacheSize int `json:"MAX_CACHE_SIZE"` // In MB, for max cached exhausted/metadata files(plus remote-raw if applicable), 0 means no limit + MaxConcurrentConversions int `json:"MAX_CONCURRENT_CONVERSIONS"` // Maximum number of concurrent image conversions + MemoryLimitMB int `json:"MEMORY_LIMIT_MB"` // Memory limit for conversions in MB } func NewWebPConfig() *WebpConfig { @@ -137,7 +141,9 @@ func NewWebPConfig() *WebpConfig { DisableKeepalive: false, CacheTTL: 259200, - MaxCacheSize: 0, + MaxCacheSize: 0, + MaxConcurrentConversions: 8, + MemoryLimitMB: 200, } } @@ -299,6 +305,24 @@ func LoadConfig() { } } + if os.Getenv("WEBP_MAX_CONCURRENT_CONVERSIONS") != "" { + maxConcurrent, err := strconv.Atoi(os.Getenv("WEBP_MAX_CONCURRENT_CONVERSIONS")) + if err != nil { + log.Warnf("WEBP_MAX_CONCURRENT_CONVERSIONS is not a valid integer, using value in config.json %d", Config.MaxConcurrentConversions) + } else { + Config.MaxConcurrentConversions = maxConcurrent + } + } + + if os.Getenv("WEBP_MEMORY_LIMIT_MB") != "" { + memLimit, err := strconv.Atoi(os.Getenv("WEBP_MEMORY_LIMIT_MB")) + if err != nil { + log.Warnf("WEBP_MEMORY_LIMIT_MB is not a valid integer, using value in config.json %d", Config.MemoryLimitMB) + } else { + Config.MemoryLimitMB = memLimit + } + } + if Config.AllowedTypes[0] == "*" { AllowAllExtensions = true } diff --git a/encoder/encoder.go b/encoder/encoder.go index 5b56e5ac..56a2cbc7 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -42,67 +42,19 @@ func loadImage(filename string) (*vips.ImageRef, error) { } func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) { - // Wait for the conversion to complete and return the converted image - retryDelay := 100 * time.Millisecond // Initial retry delay - - for { - if _, found := config.ConvertLock.Get(rawPath); found { - log.Debugf("file %s is locked under conversion, retrying in %s", rawPath, retryDelay) - time.Sleep(retryDelay) - } else { - // The lock is released, indicating that the conversion is complete - break - } - } - - // If there is a lock here, it means that another thread is converting the same image - // Lock rawPath to prevent concurrent conversion - config.ConvertLock.Set(rawPath, true, -1) - defer config.ConvertLock.Delete(rawPath) - - var wg sync.WaitGroup - wg.Add(3) - if !helper.ImageExists(avifPath) && config.Config.EnableAVIF && supportedFormats["avif"] { - go func() { - err := convertImage(rawPath, avifPath, "avif", extraParams) - if err != nil { - log.Errorln(err) - } - defer wg.Done() - }() - } else { - wg.Done() - } - - if !helper.ImageExists(webpPath) && config.Config.EnableWebP && supportedFormats["webp"] { - go func() { - err := convertImage(rawPath, webpPath, "webp", extraParams) - if err != nil { - log.Errorln(err) - } - defer wg.Done() - }() - } else { - wg.Done() - } - - if !helper.ImageExists(jxlPath) && config.Config.EnableJXL && supportedFormats["jxl"] { - go func() { - err := convertImage(rawPath, jxlPath, "jxl", extraParams) - if err != nil { - log.Errorln(err) - } - defer wg.Done() - }() - } else { - wg.Done() - } - - wg.Wait() - - if c != nil { - c <- 1 + // 使用内存管理器来控制并发 + memManager := GetMemoryManager() + job := &ConversionJob{ + RawPath: rawPath, + JxlPath: jxlPath, + AvifPath: avifPath, + WebpPath: webpPath, + ExtraParams: extraParams, + SupportedFormats: supportedFormats, + Chan: c, } + + memManager.SubmitJob(job) } func convertImage(rawPath, optimizedPath, imageType string, extraParams config.ExtraParams) error { @@ -310,3 +262,68 @@ func convertLog(itype, rawPath string, optimizedPath string, quality int) { log.Infof("%s@%d%%: %s->%s %d->%d %.2f%% deflated", itype, quality, rawPath, optimizedPath, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100) } + +// convertSync 同步转换函数，用于MemoryManager +func convertSync(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) { + // Wait for the conversion to complete and return the converted image + retryDelay := 100 * time.Millisecond // Initial retry delay + + for { + if _, found := config.ConvertLock.Get(rawPath); found { + log.Debugf("file %s is locked under conversion, retrying in %s", rawPath, retryDelay) + time.Sleep(retryDelay) + } else { + // The lock is released, indicating that the conversion is complete + break + } + } + + // If there is a lock here, it means that another thread is converting the same image + // Lock rawPath to prevent concurrent conversion + config.ConvertLock.Set(rawPath, true, -1) + defer config.ConvertLock.Delete(rawPath) + + var wg sync.WaitGroup + wg.Add(3) + if !helper.ImageExists(avifPath) && config.Config.EnableAVIF && supportedFormats["avif"] { + go func() { + err := convertImage(rawPath, avifPath, "avif", extraParams) + if err != nil { + log.Errorln(err) + } + defer wg.Done() + }() + } else { + wg.Done() + } + + if !helper.ImageExists(webpPath) && config.Config.EnableWebP && supportedFormats["webp"] { + go func() { + err := convertImage(rawPath, webpPath, "webp", extraParams) + if err != nil { + log.Errorln(err) + } + defer wg.Done() + }() + } else { + wg.Done() + } + + if !helper.ImageExists(jxlPath) && config.Config.EnableJXL && supportedFormats["jxl"] { + go func() { + err := convertImage(rawPath, jxlPath, "jxl", extraParams) + if err != nil { + log.Errorln(err) + } + defer wg.Done() + }() + } else { + wg.Done() + } + + wg.Wait() + + if c != nil { + c <- 1 + } +} diff --git a/encoder/memory_manager.go b/encoder/memory_manager.go new file mode 100644 index 00000000..2ad39e45 --- /dev/null +++ b/encoder/memory_manager.go @@ -0,0 +1,166 @@ +package encoder + +import ( + "runtime" + "sync" + "time" + "webp_server_go/config" + + log "github.com/sirupsen/logrus" +) + +// MemoryManager 管理内存和并发限制 +type MemoryManager struct { + maxConcurrency int + currentJobs int + jobQueue chan *ConversionJob + mu sync.RWMutex + semaphore chan struct{} + memoryLimitMB int64 + currentMemory int64 + memoryMu sync.RWMutex +} + +// ConversionJob 转换任务 +type ConversionJob struct { + RawPath string + JxlPath string + AvifPath string + WebpPath string + ExtraParams ExtraParams + SupportedFormats map[string]bool + Chan chan int +} + +var memManager *MemoryManager +var once sync.Once + +// GetMemoryManager 获取内存管理器单例 +func GetMemoryManager() *MemoryManager { + once.Do(func() { + // 使用配置中的参数 + maxConc := config.Config.MaxConcurrentConversions + memLimit := int64(config.Config.MemoryLimitMB) + + // 确保参数在合理范围内 + if maxConc <= 0 { + maxConc = runtime.NumCPU() * 2 // 默认每个CPU核心2个并发 + } + if maxConc > 16 { + maxConc = 16 // 最大16个并发，防止内存爆炸 + } + if maxConc < 2 { + maxConc = 2 // 最少2个并发 + } + + if memLimit <= 0 { + memLimit = 200 // 默认200MB + } + + memManager = &MemoryManager{ + maxConcurrency: maxConc, + jobQueue: make(chan *ConversionJob, maxConc*2), // 队列大小是并发数的2倍 + semaphore: make(chan struct{}, maxConc), + memoryLimitMB: memLimit, + } + + // 启动工作池 + for i := 0; i < maxConc; i++ { + go memManager.worker() + } + + log.Infof("MemoryManager initialized: max_concurrency=%d, memory_limit=%dMB", maxConc, memLimit) + }) + return memManager +} + +// worker 工作协程 +func (m *MemoryManager) worker() { + for job := range m.jobQueue { + // 获取信号量 + m.semaphore <- struct{}{} + + // 检查内存限制 + if !m.checkMemoryLimit() { + log.Warn("Memory limit reached, waiting...") + // 等待内存释放 + for !m.checkMemoryLimit() { + time.Sleep(100 * time.Millisecond) + runtime.GC() // 强制垃圾回收 + } + } + + // 记录开始 + m.mu.Lock() + m.currentJobs++ + m.mu.Unlock() + + // 估算内存使用并记录 + estimatedMemory := m.estimateMemoryUsage(job.RawPath) + m.memoryMu.Lock() + m.currentMemory += estimatedMemory + m.memoryMu.Unlock() + + // 执行转换 + m.processJob(job) + + // 释放信号量和内存计数 + <-m.semaphore + + m.mu.Lock() + m.currentJobs-- + m.mu.Unlock() + + m.memoryMu.Lock() + m.currentMemory -= estimatedMemory + m.memoryMu.Unlock() + + log.Debugf("Job completed. Current jobs: %d, Memory usage: %dMB", + m.currentJobs, m.currentMemory) + } +} + +// checkMemoryLimit 检查内存使用是否超限 +func (m *MemoryManager) checkMemoryLimit() bool { + m.memoryMu.RLock() + defer m.memoryMu.RUnlock() + return m.currentMemory < m.memoryLimitMB +} + +// estimateMemoryUsage 估算转换任务的内存使用量(MB) +func (m *MemoryManager) estimateMemoryUsage(filepath string) int64 { + // 基于文件大小的简单估算 + // 假设转换过程中内存使用是文件大小的5-10倍 + // 这里使用保守估计：20MB + return 20 +} + +// processJob 处理单个转换任务 +func (m *MemoryManager) processJob(job *ConversionJob) { + // 执行实际的转换逻辑 + convertSync(job.RawPath, job.JxlPath, job.AvifPath, job.WebpPath, + job.ExtraParams, job.SupportedFormats, job.Chan) +} + +// SubmitJob 提交转换任务 +func (m *MemoryManager) SubmitJob(job *ConversionJob) { + select { + case m.jobQueue <- job: + log.Debugf("Job submitted to queue: %s", job.RawPath) + default: + // 队列满，直接返回错误 + log.Warn("Job queue is full, rejecting job") + if job.Chan != nil { + close(job.Chan) + } + } +} + +// GetStats 获取统计信息 +func (m *MemoryManager) GetStats() (int, int64, int) { + m.mu.RLock() + m.memoryMu.RLock() + defer m.mu.RUnlock() + defer m.memoryMu.RUnlock() + return m.currentJobs, m.currentMemory, cap(m.jobQueue) - len(m.jobQueue) +} \ No newline at end of file diff --git a/encoder/prefetch.go b/encoder/prefetch.go index b222b947..cef0f60c 100644 --- a/encoder/prefetch.go +++ b/encoder/prefetch.go @@ -14,14 +14,10 @@ import ( ) func PrefetchImages() { - // maximum ongoing prefetch is depending on your core of CPU + // maximum ongoing prefetch is depending on your core of CPU and config var sTime = time.Now() - log.Infof("Prefetching using %d cores", config.Jobs) - var finishChan = make(chan int, config.Jobs) - for range config.Jobs { - finishChan <- 1 - } - + memManager := GetMemoryManager() + //prefetch, recursive through the dir all := helper.FileCount(config.Config.ImgPath) var bar = progressbar.Default(all, "Prefetching...") @@ -62,8 +58,21 @@ func PrefetchImages() { "jxl": true, } - go ConvertFilter(picAbsPath, jxlAbsPath, avifAbsPath, webpAbsPath, config.ExtraParams{Width: 0, Height: 0}, supported, finishChan) - _ = bar.Add(<-finishChan) + // 使用内存管理器进行预转换 + finishChan := make(chan int, 1) + job := &ConversionJob{ + RawPath: picAbsPath, + JxlPath: jxlAbsPath, + AvifPath: avifAbsPath, + WebpPath: webpAbsPath, + ExtraParams: config.ExtraParams{Width: 0, Height: 0}, + SupportedFormats: supported, + Chan: finishChan, + } + + memManager.SubmitJob(job) + <-finishChan + _ = bar.Add(1) return nil }) @@ -72,5 +81,4 @@ func PrefetchImages() { } elapsed := time.Since(sTime) _, _ = fmt.Fprintf(os.Stdout, "Prefetch complete in %s\n\n", elapsed) - } diff --git a/handler/router.go b/handler/router.go index 50a81c09..75ff1f05 100644 --- a/handler/router.go +++ b/handler/router.go @@ -1,6 +1,7 @@ package handler import ( + "fmt" "net/http" "net/url" "regexp" @@ -219,5 +220,14 @@ func Convert(c *fiber.Ctx) error { c.Set("Content-Type", contentType) c.Set("X-Compression-Rate", helper.GetCompressionRate(rawImageAbs, finalFilename)) + + // 添加内存管理状态到响应头 + if memManager := encoder.GetMemoryManager(); memManager != nil { + currentJobs, currentMemory, queueSize := memManager.GetStats() + c.Set("X-Memory-Jobs", fmt.Sprintf("%d", currentJobs)) + c.Set("X-Memory-Usage-MB", fmt.Sprintf("%d", currentMemory)) + c.Set("X-Queue-Size", fmt.Sprintf("%d", queueSize)) + } + return c.SendFile(finalFilename) } From c2606c45748775fe2aff6d4fd9a7eaedfa3e2aa3 Mon Sep 17 00:00:00 2001 From: szsk Date: Mon, 22 Dec 2025 11:11:01 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E4=BC=98=E5=8C=96Dockerfile=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E5=B9=B6=E6=9B=B4=E6=96=B0=E5=86=85=E5=AD=98=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E7=B1=BB=E5=9E=8B=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 7 ++++++- encoder/memory_manager.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e54fb1e2..314ea90b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,12 +8,17 @@ RUN cd /build && go mod download COPY . /build RUN cd /build && sed -i "s|.\/pics|${IMG_PATH}|g" config.json \ - && sed -i "s|\"\"|\"${EXHAUST_PATH}\"|g" config.json \ + && sed -i "s|\"./exhaust\"|\"${EXHAUST_PATH}\"|g" config.json \ && sed -i 's/127.0.0.1/0.0.0.0/g' config.json \ && go build -ldflags="-s -w" -o webp-server . FROM debian:trixie-slim +# 设置国内镜像 +RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list + RUN apt update && apt install --no-install-recommends libvips ca-certificates libjemalloc2 libtcmalloc-minimal4 curl libheif-plugin-aomenc libheif-plugin-aomdec -y && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives/* COPY --from=builder /build/webp-server /usr/bin/webp-server diff --git a/encoder/memory_manager.go b/encoder/memory_manager.go index 2ad39e45..02706a4f 100644 --- a/encoder/memory_manager.go +++ b/encoder/memory_manager.go @@ -27,7 +27,7 @@ type ConversionJob struct { JxlPath string AvifPath string WebpPath string - ExtraParams ExtraParams + ExtraParams config.ExtraParams SupportedFormats map[string]bool Chan chan int } From 7b694ae63c8e89b32d8e8558ef2829442f71d5fe Mon Sep 17 00:00:00 2001 From: szsk Date: Mon, 22 Dec 2025 11:12:34 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E8=85=BE=E8=AE=AF?= =?UTF-8?q?=E4=BA=91=E9=95=9C=E5=83=8F=E6=BA=90=E4=BC=98=E5=8C=96Dockerfil?= =?UTF-8?q?e=E6=9E=84=E5=BB=BA=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 314ea90b..75f36a9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,12 @@ FROM golang:1.25-trixie AS builder ARG IMG_PATH=/opt/pics ARG EXHAUST_PATH=/opt/exhaust + +# 设置国内镜像 +RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list + RUN apt update && apt install --no-install-recommends libvips-dev -y && mkdir /build COPY go.mod /build RUN cd /build && go mod download From c3dfdf10573ef32334b5366ed2468a471d07f47a Mon Sep 17 00:00:00 2001 From: szsk Date: Mon, 22 Dec 2025 11:44:49 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=E9=94=99=E8=AF=AF=E5=A4=84=E7=90=86=E5=B9=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=B5=84=E6=BA=90=E9=87=8A=E6=94=BE=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- encoder/encoder.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/encoder/encoder.go b/encoder/encoder.go index 56a2cbc7..9c815830 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -83,7 +83,18 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E // Image is only opened here img, err := loadImage(rawPath) - defer img.Close() + if img != nil { + defer img.Close() + } + + if err != nil || img == nil { + log.Errorf("Failed to load image %s: %v", rawPath, err) + // Notify channel that conversion is complete (with failure) + if c != nil { + c <- 1 + } + return + } // Pre-process image(auto rotate, resize, etc.) err = preProcessImage(img, imageType, extraParams) From 046ec0036bb18c73c06dfab3743f3fdc41e81d30 Mon Sep 17 00:00:00 2001 From: szsk Date: Mon, 22 Dec 2025 12:39:17 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E5=A4=84=E7=90=86=E9=94=99=E8=AF=AF=E5=A4=84=E7=90=86=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E5=B9=B6=E6=B7=BB=E5=8A=A0SVG=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=B7=B3=E8=BF=87=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- MEMORY_OPTIMIZATION_SOLUTION.md | 185 -------------------------------- encoder/encoder.go | 6 +- encoder/prefetch.go | 28 ++++- 3 files changed, 27 insertions(+), 192 deletions(-) delete mode 100644 MEMORY_OPTIMIZATION_SOLUTION.md diff --git a/MEMORY_OPTIMIZATION_SOLUTION.md b/MEMORY_OPTIMIZATION_SOLUTION.md deleted file mode 100644 index f53d2f6c..00000000 --- a/MEMORY_OPTIMIZATION_SOLUTION.md +++ /dev/null @@ -1,185 +0,0 @@ -# WebP服务器内存优化解决方案 - -## 问题分析 - -原WebP服务器在处理并发转换请求时存在以下问题： - -1. **无限制并发**：所有转换请求同时启动，没有并发控制 -2. **内存泄漏风险**：libvips在高并发下内存管理不当 -3. **OOM风险**：5个并发请求可能导致内存使用超过500MB -4. **阻塞请求**：转换过程阻塞后续请求处理 - -## 解决方案 - -### 1. 内存管理器 (encoder/memory_manager.go) - -创建了专门的内存管理器来控制并发转换： - -- **并发限制**：限制最大同时转换数量（默认6个） -- **内存限制**：设置转换过程内存使用上限（默认150MB） -- **任务队列**：使用有界队列缓存待转换任务 -- **工作池模式**：使用固定数量的工作协程处理转换任务 - -```go -type MemoryManager struct { - maxConcurrency int - currentJobs int - jobQueue chan *ConversionJob - semaphore chan struct{} - memoryLimitMB int64 - currentMemory int64 -} -``` - -### 2. 转换流程优化 (encoder/encoder.go) - -将原有的`ConvertFilter`函数重构： - -- **异步提交**：转换任务提交给内存管理器 -- **同步处理**：内存管理器内部使用工作池同步处理 -- **资源控制**：通过信号量控制并发数量 - -### 3. 配置参数增强 (config/config.go) - -新增配置参数： - -- `MAX_CONCURRENT_CONVERSIONS`: 最大并发转换数（默认6） -- `MEMORY_LIMIT_MB`: 转换内存限制（默认150MB） - -### 4. 预转换优化 (encoder/prefetch.go) - -预转换过程使用内存管理器： - -- **队列化处理**：预转换任务通过队列处理 -- **内存控制**：避免预转换时内存爆炸 -- **进度监控**：保持原有的进度条功能 - -### 5. 状态监控 (handler/router.go) - -添加内存使用状态监控： - -- **响应头信息**：`X-Memory-Jobs`, `X-Memory-Usage-MB`, `X-Queue-Size` -- **实时状态**：客户端可以监控服务器内存状态 -- **调试支持**：便于运维人员监控和调试 - -## 配置说明 - -### config.json 示例 - -```json -{ - "HOST": "127.0.0.1", - "PORT": "3333", - "QUALITY": "80", - "IMG_PATH": "./pics", - "EXHAUST_PATH": "./exhaust", - "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], - "CONVERT_TYPES": ["webp"], - "STRIP_METADATA": true, - "ENABLE_EXTRA_PARAMS": false, - "READ_BUFFER_SIZE": 4096, - "CONCURRENCY": 262144, - "DISABLE_KEEPALIVE": false, - "CACHE_TTL": 259200, - "MAX_CACHE_SIZE": 0, - "MAX_CONCURRENT_CONVERSIONS": 6, - "MEMORY_LIMIT_MB": 150 -} -``` - -### 环境变量支持 - -```bash -# 最大并发转换数 -export WEBP_MAX_CONCURRENT_CONVERSIONS=6 - -# 内存限制（MB） -export WEBP_MEMORY_LIMIT_MB=150 -``` - -## 性能改进效果 - -### 内存使用控制 - -- **原来**：5个并发请求 → >500MB内存 -- **现在**：最多6个并发，内存限制150MB -- **OOM风险**：从高风险 → 低风险 - -### 并发控制 - -- **原来**：无限制并发，系统过载 -- **现在**：有界队列 + 工作池，平滑处理 - -### 请求处理 - -- **原来**：转换请求阻塞后续请求 -- **现在**：任务队列化，非阻塞提交 - -## 监控和调试 - -### 响应头监控 - -每个响应都会包含内存状态信息： - -``` -X-Memory-Jobs: 2 # 当前正在处理的转换任务数 -X-Memory-Usage-MB: 45 # 当前预估内存使用量(MB) -X-Queue-Size: 3 # 队列中等待的任务数 -``` - -### 日志监控 - -系统会输出详细的内存管理日志： - -``` -INFO[0001] MemoryManager initialized: max_concurrency=6, memory_limit=150MB -DEBUG[0002] Job submitted to queue: /path/to/image.jpg -DEBUG[0003] Job completed. Current jobs: 2, Memory usage: 45MB -``` - -## 部署建议 - -### 1GB内存服务器配置 - -```json -{ - "MAX_CONCURRENT_CONVERSIONS": 4, - "MEMORY_LIMIT_MB": 120 -} -``` - -### 2GB内存服务器配置 - -```json -{ - "MAX_CONCURRENT_CONVERSIONS": 8, - "MEMORY_LIMIT_MB": 200 -} -``` - -## 使用方式 - -### 编译和运行 - -```bash -go build -o webp-server . -./webp-server -config config.json -``` - -### 预转换模式 - -```bash -# 后台预转换 -./webp-server -prefetch -config config.json - -# 前台预转换完成后退出 -./webp-server -prefetch-foreground -config config.json -``` - -## 验证方法 - -1. **内存监控**：使用`htop`或`top`观察内存使用 -2. **并发测试**：使用`ab`或`wrk`进行并发请求测试 -3. **状态检查**：检查响应头中的内存状态信息 - -这个解决方案通过引入内存管理器和并发控制，有效解决了WebP服务器在高并发场景下的内存消耗问题，确保服务器在1GB内存的虚拟机上稳定运行。 \ No newline at end of file diff --git a/encoder/encoder.go b/encoder/encoder.go index 9c815830..3552ceb4 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -89,11 +89,7 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E if err != nil || img == nil { log.Errorf("Failed to load image %s: %v", rawPath, err) - // Notify channel that conversion is complete (with failure) - if c != nil { - c <- 1 - } - return + return err } // Pre-process image(auto rotate, resize, etc.) diff --git a/encoder/prefetch.go b/encoder/prefetch.go index cef0f60c..35909bfc 100644 --- a/encoder/prefetch.go +++ b/encoder/prefetch.go @@ -5,6 +5,7 @@ import ( "os" "path" "path/filepath" + "strings" "time" "webp_server_go/config" "webp_server_go/helper" @@ -29,6 +30,13 @@ func PrefetchImages() { if info.IsDir() { return nil } + // Skip SVG files as they don't need WebP conversion and often cause dimension errors + if strings.ToLower(filepath.Ext(picAbsPath)) == ".svg" { + log.Infof("Skipping SVG file: %s", picAbsPath) + _ = bar.Add(1) + return nil + } + // Only convert files with image extensions, use smaller of config.DefaultAllowedTypes and config.Config.AllowedTypes if helper.CheckAllowedExtension(picAbsPath) { // File type is allowed by user, check if it is an image @@ -70,9 +78,25 @@ func PrefetchImages() { Chan: finishChan, } + // Add error recovery mechanism + defer func() { + if r := recover(); r != nil { + log.Errorf("Recovered from panic while processing %s: %v", picAbsPath, r) + _ = bar.Add(1) + } + }() + memManager.SubmitJob(job) - <-finishChan - _ = bar.Add(1) + + // Add timeout to prevent hanging + select { + case <-finishChan: + _ = bar.Add(1) + case <-time.After(30 * time.Second): + log.Warnf("Timeout processing %s after 30 seconds, skipping...", picAbsPath) + _ = bar.Add(1) + } + return nil }) From 16178558f845973401ef10f018b92a6e3a775623 Mon Sep 17 00:00:00 2001 From: szsk Date: Mon, 22 Dec 2025 11:32:40 +0800 Subject: [PATCH 07/10] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E5=AD=98=E5=82=A8=E8=B7=AF=E5=BE=84=E5=B9=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=9D=E5=AF=B9=E8=B7=AF=E5=BE=84=E5=A4=84?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile.fixed | 35 +++++++++++++++++++++++++++++++++++ config.json | 4 ++-- helper/metadata.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.fixed diff --git a/Dockerfile.fixed b/Dockerfile.fixed new file mode 100644 index 00000000..f8af9f32 --- /dev/null +++ b/Dockerfile.fixed @@ -0,0 +1,35 @@ +FROM golang:1.25-trixie AS builder + +ARG IMG_PATH=/opt/pics +ARG EXHAUST_PATH=/opt/exhaust + +# 设置国内镜像 +RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list + +RUN apt update && apt install --no-install-recommends libvips-dev -y && mkdir /build +COPY go.mod /build +RUN cd /build && go mod download + +COPY . /build +RUN cd /build && sed -i 's|"\./pics"|"'${IMG_PATH}'"|g' config.json \ + && sed -i 's|"\./exhaust"|"'${EXHAUST_PATH}'"|g' config.json \ + && sed -i 's/127.0.0.1/0.0.0.0/g' config.json \ + && go build -ldflags="-s -w" -o webp-server . + +FROM debian:trixie-slim + +# 设置国内镜像 +RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list + +RUN apt update && apt install --no-install-recommends libvips ca-certificates libjemalloc2 libtcmalloc-minimal4 curl libheif-plugin-aomenc libheif-plugin-aomdec -y && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives/* + +COPY --from=builder /build/webp-server /usr/bin/webp-server +COPY --from=builder /build/config.json /etc/config.json + +WORKDIR /opt +VOLUME /opt/exhaust +CMD ["/usr/bin/webp-server", "--config", "/etc/config.json"] \ No newline at end of file diff --git a/config.json b/config.json index 53934990..697b0eff 100644 --- a/config.json +++ b/config.json @@ -2,8 +2,8 @@ "HOST": "127.0.0.1", "PORT": "3333", "QUALITY": "80", - "IMG_PATH": "./pics", - "EXHAUST_PATH": "./exhaust", + "IMG_PATH": "/opt/pics", + "EXHAUST_PATH": "/opt/exhaust", "IMG_MAP": {}, "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], "CONVERT_TYPES": ["webp"], diff --git a/helper/metadata.go b/helper/metadata.go index 949e3ae0..d7abbb18 100644 --- a/helper/metadata.go +++ b/helper/metadata.go @@ -5,6 +5,7 @@ import ( "net/url" "os" "path" + "strings" "webp_server_go/config" "github.com/buckket/go-blurhash" @@ -19,6 +20,34 @@ func getId(p string, subdir string) (id string, filePath string, santizedPath st fileID := HashString(p) return fileID, path.Join(config.Config.RemoteRawPath, subdir, fileID) + path.Ext(p), "" } + + // Check if p is already an absolute path + if path.IsAbs(p) { + // For absolute paths (like in prefetch), use the path directly + parsed, _ := url.Parse(p) + width := parsed.Query().Get("width") + height := parsed.Query().Get("height") + max_width := parsed.Query().Get("max_width") + max_height := parsed.Query().Get("max_height") + + // Remove the ImgPath prefix if present to avoid duplication + relativePath := strings.TrimPrefix(p, config.Config.ImgPath) + if strings.HasPrefix(relativePath, "/") { + relativePath = relativePath[1:] // Remove leading slash + } + if relativePath == "" { + relativePath = p // Fallback to original path if trimming removes everything + } + + parsedPath := "/" + relativePath + santizedPath = parsedPath + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height + id = HashString(santizedPath) + filePath = p // Use the absolute path directly + + return id, filePath, santizedPath + } + + // For relative paths (like in web requests) parsed, _ := url.Parse(p) width := parsed.Query().Get("width") height := parsed.Query().Get("height") From bd02936b4c9b46d45b2b2332c30d1b3dc2a00a0a Mon Sep 17 00:00:00 2001 From: szsk Date: Mon, 22 Dec 2025 12:53:42 +0800 Subject: [PATCH 08/10] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=86=97=E4=BD=99?= =?UTF-8?q?=E7=9A=84Dockerfile.fixed=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile.fixed | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 Dockerfile.fixed diff --git a/Dockerfile.fixed b/Dockerfile.fixed deleted file mode 100644 index f8af9f32..00000000 --- a/Dockerfile.fixed +++ /dev/null @@ -1,35 +0,0 @@ -FROM golang:1.25-trixie AS builder - -ARG IMG_PATH=/opt/pics -ARG EXHAUST_PATH=/opt/exhaust - -# 设置国内镜像 -RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list - -RUN apt update && apt install --no-install-recommends libvips-dev -y && mkdir /build -COPY go.mod /build -RUN cd /build && go mod download - -COPY . /build -RUN cd /build && sed -i 's|"\./pics"|"'${IMG_PATH}'"|g' config.json \ - && sed -i 's|"\./exhaust"|"'${EXHAUST_PATH}'"|g' config.json \ - && sed -i 's/127.0.0.1/0.0.0.0/g' config.json \ - && go build -ldflags="-s -w" -o webp-server . - -FROM debian:trixie-slim - -# 设置国内镜像 -RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list - -RUN apt update && apt install --no-install-recommends libvips ca-certificates libjemalloc2 libtcmalloc-minimal4 curl libheif-plugin-aomenc libheif-plugin-aomdec -y && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives/* - -COPY --from=builder /build/webp-server /usr/bin/webp-server -COPY --from=builder /build/config.json /etc/config.json - -WORKDIR /opt -VOLUME /opt/exhaust -CMD ["/usr/bin/webp-server", "--config", "/etc/config.json"] \ No newline at end of file From 3ae2ede48e957b653c90ae3cbe24d7f321918dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E5=AD=90=E7=83=A7=E7=83=A4?= Date: Mon, 22 Dec 2025 13:08:56 +0800 Subject: [PATCH 09/10] Remove .cnb.yml configuration file and clean up Dockerfile by eliminating redundant mirror source settings for improved build efficiency. --- .cnb.yml | 15 --------------- Dockerfile | 12 +----------- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 .cnb.yml diff --git a/.cnb.yml b/.cnb.yml deleted file mode 100644 index 6db698fa..00000000 --- a/.cnb.yml +++ /dev/null @@ -1,15 +0,0 @@ -# .cnb.yml -$: - push: - - name: docker-build-push - services: - - docker - stages: - - name: docker login - script: docker login -u ${CNB_TOKEN_USER_NAME} -p "${CNB_TOKEN}" docker.cnb.cool - - - name: docker build - script: docker build -f Dockerfile -t docker.cnb.cool/szsk968/webp_server_go . - - - name: docker push - script: docker push docker.cnb.cool/szsk968/webp_server_go \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 75f36a9e..630b2b73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,7 @@ FROM golang:1.25-trixie AS builder ARG IMG_PATH=/opt/pics ARG EXHAUST_PATH=/opt/exhaust - -# 设置国内镜像 -RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list - + RUN apt update && apt install --no-install-recommends libvips-dev -y && mkdir /build COPY go.mod /build RUN cd /build && go mod download @@ -20,11 +15,6 @@ RUN cd /build && sed -i "s|.\/pics|${IMG_PATH}|g" config.json \ FROM debian:trixie-slim -# 设置国内镜像 -RUN echo "deb http://mirrors.tencentyun.com/debian trixie main" > /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian trixie-updates main" >> /etc/apt/sources.list && \ - echo "deb http://mirrors.tencentyun.com/debian-security trixie-security main" >> /etc/apt/sources.list - RUN apt update && apt install --no-install-recommends libvips ca-certificates libjemalloc2 libtcmalloc-minimal4 curl libheif-plugin-aomenc libheif-plugin-aomdec -y && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives/* COPY --from=builder /build/webp-server /usr/bin/webp-server From 1ccd4f31710a89c1c89ff26ce791c67d014f2140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E5=AD=90=E7=83=A7=E7=83=A4?= Date: Mon, 22 Dec 2025 13:45:25 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E6=B7=87=EE=86=BC=EE=98=B2=E5=A8=B4?= =?UTF-8?q?=E5=AC=AD=E7=98=AF=E6=BE=B6=E8=BE=AB=E8=A7=A6=E9=94=9B=E6=B0=AB?= =?UTF-8?q?=E6=85=A8=E5=A7=9D=EE=96=ADetId=E9=8D=91=E8=8A=A5=E6=9A=9F?= =?UTF-8?q?=E9=90=A8=E5=8B=AD=E7=B2=B7=E7=80=B5=E7=A1=85=E7=9F=BE=E5=AF=B0?= =?UTF-8?q?=E5=8B=AB=E5=9E=BD=E9=8F=82=EE=85=A2=E2=82=AC=E6=98=8F=E7=B7=AB?= =?UTF-8?q?=E9=AA=9E=E8=88=B5=E4=BB=AE=E6=BE=B6=E5=B3=9Config.json?= =?UTF-8?q?=E6=A6=9B=E6=A8=BF=EE=85=BB=E7=92=BA=EE=88=9A=E7=B7=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json | 4 ++-- helper/metadata.go | 32 +++++++++++++++----------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/config.json b/config.json index 697b0eff..53934990 100644 --- a/config.json +++ b/config.json @@ -2,8 +2,8 @@ "HOST": "127.0.0.1", "PORT": "3333", "QUALITY": "80", - "IMG_PATH": "/opt/pics", - "EXHAUST_PATH": "/opt/exhaust", + "IMG_PATH": "./pics", + "EXHAUST_PATH": "./exhaust", "IMG_MAP": {}, "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], "CONVERT_TYPES": ["webp"], diff --git a/helper/metadata.go b/helper/metadata.go index d7abbb18..1f209d27 100644 --- a/helper/metadata.go +++ b/helper/metadata.go @@ -21,38 +21,36 @@ func getId(p string, subdir string) (id string, filePath string, santizedPath st return fileID, path.Join(config.Config.RemoteRawPath, subdir, fileID) + path.Ext(p), "" } - // Check if p is already an absolute path - if path.IsAbs(p) { + // Parse URL first to separate path and query + parsed, _ := url.Parse(p) + width := parsed.Query().Get("width") + height := parsed.Query().Get("height") + max_width := parsed.Query().Get("max_width") + max_height := parsed.Query().Get("max_height") + + // Check if the path part is a true absolute filesystem path (not just starting with /) + // True absolute paths are like /opt/pics/image.jpg (used in prefetch) + // Web request paths like /image.jpg should be treated as relative paths + if path.IsAbs(parsed.Path) && strings.HasPrefix(parsed.Path, config.Config.ImgPath) { // For absolute paths (like in prefetch), use the path directly - parsed, _ := url.Parse(p) - width := parsed.Query().Get("width") - height := parsed.Query().Get("height") - max_width := parsed.Query().Get("max_width") - max_height := parsed.Query().Get("max_height") - - // Remove the ImgPath prefix if present to avoid duplication - relativePath := strings.TrimPrefix(p, config.Config.ImgPath) + // Remove the ImgPath prefix to get relative path + relativePath := strings.TrimPrefix(parsed.Path, config.Config.ImgPath) if strings.HasPrefix(relativePath, "/") { relativePath = relativePath[1:] // Remove leading slash } if relativePath == "" { - relativePath = p // Fallback to original path if trimming removes everything + relativePath = parsed.Path // Fallback to original path if trimming removes everything } parsedPath := "/" + relativePath santizedPath = parsedPath + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height id = HashString(santizedPath) - filePath = p // Use the absolute path directly + filePath = parsed.Path // Use the absolute path directly return id, filePath, santizedPath } // For relative paths (like in web requests) - parsed, _ := url.Parse(p) - width := parsed.Query().Get("width") - height := parsed.Query().Get("height") - max_width := parsed.Query().Get("max_width") - max_height := parsed.Query().Get("max_height") // santizedPath will be /webp_server.jpg?width=200\u0026height=\u0026max_width=\u0026max_height= in local mode when requesting /webp_server.jpg?width=200 // santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh santizedPath = parsed.Path + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height