内存管理优化、错误处理优化、构建流程改进和路径处理优化

Dockerfile

@@ -2,13 +2,14 @@ FROM golang:1.25-trixie AS builder
 
 ARG IMG_PATH=/opt/pics
 ARG EXHAUST_PATH=/opt/exhaust
+
 RUN apt update && apt install --no-install-recommends libvips-dev -y && mkdir /build
 COPY go.mod /build
 RUN cd /build && go mod download
 
 COPY . /build
 RUN cd /build && sed -i "s|.\/pics|${IMG_PATH}|g" config.json  \
-    && sed -i "s|\"\"|\"${EXHAUST_PATH}\"|g" config.json  \
+    && sed -i "s|\"./exhaust\"|\"${EXHAUST_PATH}\"|g" config.json  \
     && sed -i 's/127.0.0.1/0.0.0.0/g' config.json  \
     && go build -ldflags="-s -w" -o webp-server .
 

config.json

@@ -12,5 +12,8 @@
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
   "DISABLE_KEEPALIVE": false,
-  "CACHE_TTL": 259200
+  "CACHE_TTL": 259200,
+  "MAX_CACHE_SIZE": 0,
+  "MAX_CONCURRENT_CONVERSIONS": 6,
+  "MEMORY_LIMIT_MB": 150
 }

config/config.go

@@ -38,7 +38,9 @@ const (
   "CONCURRENCY": 262144,
   "DISABLE_KEEPALIVE": false,
   "CACHE_TTL": 259200,
-  "MAX_CACHE_SIZE": 0
+  "MAX_CACHE_SIZE": 0,
+  "MAX_CONCURRENT_CONVERSIONS": 8,
+  "MEMORY_LIMIT_MB": 200
 }`
 )
 
@@ -106,7 +108,9 @@ type WebpConfig struct {
 	DisableKeepalive bool `json:"DISABLE_KEEPALIVE"`
 	CacheTTL         int  `json:"CACHE_TTL"` // In minutes
 
-	MaxCacheSize int `json:"MAX_CACHE_SIZE"` // In MB, for max cached exhausted/metadata files(plus remote-raw if applicable), 0 means no limit
+	MaxCacheSize             int `json:"MAX_CACHE_SIZE"`             // In MB, for max cached exhausted/metadata files(plus remote-raw if applicable), 0 means no limit
+	MaxConcurrentConversions int `json:"MAX_CONCURRENT_CONVERSIONS"` // Maximum number of concurrent image conversions
+	MemoryLimitMB            int `json:"MEMORY_LIMIT_MB"`            // Memory limit for conversions in MB
 }
 
 func NewWebPConfig() *WebpConfig {
@@ -137,7 +141,9 @@ func NewWebPConfig() *WebpConfig {
 		DisableKeepalive:           false,
 		CacheTTL:                   259200,
 
-		MaxCacheSize: 0,
+		MaxCacheSize:              0,
+		MaxConcurrentConversions: 8,
+		MemoryLimitMB:            200,
 	}
 }
 
@@ -299,6 +305,24 @@ func LoadConfig() {
 		}
 	}
 
+	if os.Getenv("WEBP_MAX_CONCURRENT_CONVERSIONS") != "" {
+		maxConcurrent, err := strconv.Atoi(os.Getenv("WEBP_MAX_CONCURRENT_CONVERSIONS"))
+		if err != nil {
+			log.Warnf("WEBP_MAX_CONCURRENT_CONVERSIONS is not a valid integer, using value in config.json %d", Config.MaxConcurrentConversions)
+		} else {
+			Config.MaxConcurrentConversions = maxConcurrent
+		}
+	}
+
+	if os.Getenv("WEBP_MEMORY_LIMIT_MB") != "" {
+		memLimit, err := strconv.Atoi(os.Getenv("WEBP_MEMORY_LIMIT_MB"))
+		if err != nil {
+			log.Warnf("WEBP_MEMORY_LIMIT_MB is not a valid integer, using value in config.json %d", Config.MemoryLimitMB)
+		} else {
+			Config.MemoryLimitMB = memLimit
+		}
+	}
+
 	if Config.AllowedTypes[0] == "*" {
 		AllowAllExtensions = true
 	}

encoder/encoder.go

@@ -42,67 +42,19 @@ func loadImage(filename string) (*vips.ImageRef, error) {
 }
 
 func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) {
-	// Wait for the conversion to complete and return the converted image
-	retryDelay := 100 * time.Millisecond // Initial retry delay
-
-	for {
-		if _, found := config.ConvertLock.Get(rawPath); found {
-			log.Debugf("file %s is locked under conversion, retrying in %s", rawPath, retryDelay)
-			time.Sleep(retryDelay)
-		} else {
-			// The lock is released, indicating that the conversion is complete
-			break
-		}
-	}
-
-	// If there is a lock here, it means that another thread is converting the same image
-	// Lock rawPath to prevent concurrent conversion
-	config.ConvertLock.Set(rawPath, true, -1)
-	defer config.ConvertLock.Delete(rawPath)
-
-	var wg sync.WaitGroup
-	wg.Add(3)
-	if !helper.ImageExists(avifPath) && config.Config.EnableAVIF && supportedFormats["avif"] {
-		go func() {
-			err := convertImage(rawPath, avifPath, "avif", extraParams)
-			if err != nil {
-				log.Errorln(err)
-			}
-			defer wg.Done()
-		}()
-	} else {
-		wg.Done()
-	}
-
-	if !helper.ImageExists(webpPath) && config.Config.EnableWebP && supportedFormats["webp"] {
-		go func() {
-			err := convertImage(rawPath, webpPath, "webp", extraParams)
-			if err != nil {
-				log.Errorln(err)
-			}
-			defer wg.Done()
-		}()
-	} else {
-		wg.Done()
-	}
-
-	if !helper.ImageExists(jxlPath) && config.Config.EnableJXL && supportedFormats["jxl"] {
-		go func() {
-			err := convertImage(rawPath, jxlPath, "jxl", extraParams)
-			if err != nil {
-				log.Errorln(err)
-			}
-			defer wg.Done()
-		}()
-	} else {
-		wg.Done()
-	}
-
-	wg.Wait()
-
-	if c != nil {
-		c <- 1
+	// 使用内存管理器来控制并发
+	memManager := GetMemoryManager()
+	job := &ConversionJob{
+		RawPath:       rawPath,
+		JxlPath:       jxlPath,
+		AvifPath:      avifPath,
+		WebpPath:      webpPath,
+		ExtraParams:   extraParams,
+		SupportedFormats: supportedFormats,
+		Chan:          c,
 	}
+
+	memManager.SubmitJob(job)
 }
 
 func convertImage(rawPath, optimizedPath, imageType string, extraParams config.ExtraParams) error {
@@ -131,7 +83,14 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E
 
 	// Image is only opened here
 	img, err := loadImage(rawPath)
-	defer img.Close()
+	if img != nil {
+		defer img.Close()
+	}
+
+	if err != nil || img == nil {
+		log.Errorf("Failed to load image %s: %v", rawPath, err)
+		return err
+	}
 
 	// Pre-process image(auto rotate, resize, etc.)
 	err = preProcessImage(img, imageType, extraParams)
@@ -310,3 +269,68 @@ func convertLog(itype, rawPath string, optimizedPath string, quality int) {
 	log.Infof("%s@%d%%: %s->%s %d->%d %.2f%% deflated", itype, quality,
 		rawPath, optimizedPath, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100)
 }
+
+// convertSync 同步转换函数，用于MemoryManager
+func convertSync(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) {
+	// Wait for the conversion to complete and return the converted image
+	retryDelay := 100 * time.Millisecond // Initial retry delay
+
+	for {
+		if _, found := config.ConvertLock.Get(rawPath); found {
+			log.Debugf("file %s is locked under conversion, retrying in %s", rawPath, retryDelay)
+			time.Sleep(retryDelay)
+		} else {
+			// The lock is released, indicating that the conversion is complete
+			break
+		}
+	}
+
+	// If there is a lock here, it means that another thread is converting the same image
+	// Lock rawPath to prevent concurrent conversion
+	config.ConvertLock.Set(rawPath, true, -1)
+	defer config.ConvertLock.Delete(rawPath)
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+	if !helper.ImageExists(avifPath) && config.Config.EnableAVIF && supportedFormats["avif"] {
+		go func() {
+			err := convertImage(rawPath, avifPath, "avif", extraParams)
+			if err != nil {
+				log.Errorln(err)
+			}
+			defer wg.Done()
+		}()
+	} else {
+		wg.Done()
+	}
+
+	if !helper.ImageExists(webpPath) && config.Config.EnableWebP && supportedFormats["webp"] {
+		go func() {
+			err := convertImage(rawPath, webpPath, "webp", extraParams)
+			if err != nil {
+				log.Errorln(err)
+			}
+			defer wg.Done()
+		}()
+	} else {
+		wg.Done()
+	}
+
+	if !helper.ImageExists(jxlPath) && config.Config.EnableJXL && supportedFormats["jxl"] {
+		go func() {
+			err := convertImage(rawPath, jxlPath, "jxl", extraParams)
+			if err != nil {
+				log.Errorln(err)
+			}
+			defer wg.Done()
+		}()
+	} else {
+		wg.Done()
+	}
+
+	wg.Wait()
+
+	if c != nil {
+		c <- 1
+	}
+}