Refactor: HTTP clients, unified HTTP2/QUIC options, Apple engines

Author: nekohasekai

adapter/http.go

@@ -0,0 +1,43 @@
+package adapter
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	"github.com/sagernet/sing-box/option"
+	"github.com/sagernet/sing/common/logger"
+)
+
+type HTTPTransport interface {
+	http.RoundTripper
+	CloseIdleConnections()
+	Reset()
+}
+
+type HTTPClientManager interface {
+	ResolveTransport(ctx context.Context, logger logger.ContextLogger, options option.HTTPClientOptions) (HTTPTransport, error)
+	DefaultTransport() HTTPTransport
+	ResetNetwork()
+}
+
+type HTTPStartContext struct {
+	access     sync.Mutex
+	transports []HTTPTransport
+}
+
+func NewHTTPStartContext() *HTTPStartContext {
+	return &HTTPStartContext{}
+}
+
+func (c *HTTPStartContext) Register(transport HTTPTransport) {
+	c.access.Lock()
+	defer c.access.Unlock()
+	c.transports = append(c.transports, transport)
+}
+
+func (c *HTTPStartContext) Close() {
+	for _, transport := range c.transports {
+		transport.CloseIdleConnections()
+	}
+}

adapter/router.go

@@ -2,17 +2,11 @@ package adapter
 
 import (
 	"context"
-	"crypto/tls"
 	"net"
-	"net/http"
-	"sync"
 	"time"
 
-	C "github.com/sagernet/sing-box/constant"
 	"github.com/sagernet/sing-tun"
-	M "github.com/sagernet/sing/common/metadata"
 	N "github.com/sagernet/sing/common/network"
-	"github.com/sagernet/sing/common/ntp"
 	"github.com/sagernet/sing/common/x/list"
 
 	"go4.org/netipx"
@@ -77,46 +71,3 @@ type RuleSetMetadata struct {
 	ContainsIPCIDRRule       bool
 	ContainsDNSQueryTypeRule bool
 }
-type HTTPStartContext struct {
-	ctx             context.Context
-	access          sync.Mutex
-	httpClientCache map[string]*http.Client
-}
-
-func NewHTTPStartContext(ctx context.Context) *HTTPStartContext {
-	return &HTTPStartContext{
-		ctx:             ctx,
-		httpClientCache: make(map[string]*http.Client),
-	}
-}
-
-func (c *HTTPStartContext) HTTPClient(detour string, dialer N.Dialer) *http.Client {
-	c.access.Lock()
-	defer c.access.Unlock()
-	if httpClient, loaded := c.httpClientCache[detour]; loaded {
-		return httpClient
-	}
-	httpClient := &http.Client{
-		Transport: &http.Transport{
-			ForceAttemptHTTP2:   true,
-			TLSHandshakeTimeout: C.TCPTimeout,
-			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return dialer.DialContext(ctx, network, M.ParseSocksaddr(addr))
-			},
-			TLSClientConfig: &tls.Config{
-				Time:    ntp.TimeFuncFromContext(c.ctx),
-				RootCAs: RootPoolFromContext(c.ctx),
-			},
-		},
-	}
-	c.httpClientCache[detour] = httpClient
-	return httpClient
-}
-
-func (c *HTTPStartContext) Close() {
-	c.access.Lock()
-	defer c.access.Unlock()
-	for _, client := range c.httpClientCache {
-		client.CloseIdleConnections()
-	}
-}

box.go

@@ -16,12 +16,14 @@ import (
 	boxService "github.com/sagernet/sing-box/adapter/service"
 	"github.com/sagernet/sing-box/common/certificate"
 	"github.com/sagernet/sing-box/common/dialer"
+	"github.com/sagernet/sing-box/common/httpclient"
 	"github.com/sagernet/sing-box/common/taskmonitor"
 	"github.com/sagernet/sing-box/common/tls"
 	C "github.com/sagernet/sing-box/constant"
 	"github.com/sagernet/sing-box/dns"
 	"github.com/sagernet/sing-box/experimental"
 	"github.com/sagernet/sing-box/experimental/cachefile"
+	"github.com/sagernet/sing-box/experimental/deprecated"
 	"github.com/sagernet/sing-box/log"
 	"github.com/sagernet/sing-box/option"
 	"github.com/sagernet/sing-box/protocol/direct"
@@ -50,6 +52,7 @@ type Box struct {
 	dnsRouter           *dns.Router
 	connection          *route.ConnectionManager
 	router              *route.Router
+	httpClientService   adapter.LifecycleService
 	internalService     []adapter.LifecycleService
 	done                chan struct{}
 }
@@ -169,6 +172,7 @@ func New(options Options) (*Box, error) {
 	}
 
 	var internalServices []adapter.LifecycleService
+	routeOptions := common.PtrValueOrDefault(options.Route)
 	certificateOptions := common.PtrValueOrDefault(options.Certificate)
 	if C.IsAndroid || certificateOptions.Store != "" && certificateOptions.Store != C.CertificateStoreSystem ||
 		len(certificateOptions.Certificate) > 0 ||
@@ -181,8 +185,6 @@ func New(options Options) (*Box, error) {
 		service.MustRegister[adapter.CertificateStore](ctx, certificateStore)
 		internalServices = append(internalServices, certificateStore)
 	}
-
-	routeOptions := common.PtrValueOrDefault(options.Route)
 	dnsOptions := common.PtrValueOrDefault(options.DNS)
 	endpointManager := endpoint.NewManager(logFactory.NewLogger("endpoint"), endpointRegistry)
 	inboundManager := inbound.NewManager(logFactory.NewLogger("inbound"), inboundRegistry, endpointManager)
@@ -209,6 +211,10 @@ func New(options Options) (*Box, error) {
 	service.MustRegister[adapter.NetworkManager](ctx, networkManager)
 	connectionManager := route.NewConnectionManager(logFactory.NewLogger("connection"))
 	service.MustRegister[adapter.ConnectionManager](ctx, connectionManager)
+	// Must register after ConnectionManager: the Apple HTTP engine's proxy bridge reads it from the context when Manager.Start resolves the default client.
+	httpClientManager := httpclient.NewManager(ctx, logFactory.NewLogger("httpclient"), options.HTTPClients, routeOptions.DefaultHTTPClient)
+	service.MustRegister[adapter.HTTPClientManager](ctx, httpClientManager)
+	httpClientService := adapter.LifecycleService(httpClientManager)
 	router := route.NewRouter(ctx, logFactory, routeOptions, dnsOptions)
 	service.MustRegister[adapter.Router](ctx, router)
 	err = router.Initialize(routeOptions.Rules, routeOptions.RuleSet)
@@ -368,6 +374,12 @@ func New(options Options) (*Box, error) {
 			&option.LocalDNSServerOptions{},
 		)
 	})
+	httpClientManager.Initialize(func() (*httpclient.ManagedTransport, error) {
+		deprecated.Report(ctx, deprecated.OptionImplicitDefaultHTTPClient)
+		var httpClientOptions option.HTTPClientOptions
+		httpClientOptions.DefaultOutbound = true
+		return httpclient.NewTransport(ctx, logFactory.NewLogger("httpclient"), "", httpClientOptions)
+	})
 	if platformInterface != nil {
 		err = platformInterface.Initialize(networkManager)
 		if err != nil {
@@ -428,6 +440,7 @@ func New(options Options) (*Box, error) {
 		dnsRouter:           dnsRouter,
 		connection:          connectionManager,
 		router:              router,
+		httpClientService:   httpClientService,
 		createdAt:           createdAt,
 		logFactory:          logFactory,
 		logger:              logFactory.Logger(),
@@ -490,7 +503,15 @@ func (s *Box) preStart() error {
 	if err != nil {
 		return err
 	}
-	err = adapter.Start(s.logger, adapter.StartStateStart, s.outbound, s.dnsTransport, s.network, s.connection, s.router, s.dnsRouter)
+	err = adapter.Start(s.logger, adapter.StartStateStart, s.outbound, s.dnsTransport, s.network, s.connection)
+	if err != nil {
+		return err
+	}
+	err = adapter.StartNamed(s.logger, adapter.StartStateStart, []adapter.LifecycleService{s.httpClientService})
+	if err != nil {
+		return err
+	}
+	err = adapter.Start(s.logger, adapter.StartStateStart, s.router, s.dnsRouter)
 	if err != nil {
 		return err
 	}
@@ -567,6 +588,14 @@ func (s *Box) Close() error {
 		})
 		s.logger.Trace("close ", closeItem.name, " completed (", F.Seconds(time.Since(startTime).Seconds()), "s)")
 	}
+	if s.httpClientService != nil {
+		s.logger.Trace("close ", s.httpClientService.Name())
+		startTime := time.Now()
+		err = E.Append(err, s.httpClientService.Close(), func(err error) error {
+			return E.Cause(err, "close ", s.httpClientService.Name())
+		})
+		s.logger.Trace("close ", s.httpClientService.Name(), " completed (", F.Seconds(time.Since(startTime).Seconds()), "s)")
+	}
 	for _, lifecycleService := range s.internalService {
 		s.logger.Trace("close ", lifecycleService.Name())
 		startTime := time.Now()

cmd/internal/build_libbox/main.go

@@ -204,6 +204,9 @@ func buildApple() {
 		"-target", bindTarget,
 		"-libname=box",
 		"-tags-not-macos=with_low_memory",
+		"-iosversion=15.0",
+		"-macosversion=13.0",
+		"-tvosversion=17.0",
 	}
 	//if !withTailscale {
 	//	args = append(args, "-tags-macos="+strings.Join(memcTags, ","))

cmd/internal/update_certificates/main.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/sagernet/sing-box/log"
@@ -35,21 +36,9 @@ func updateMozillaIncludedRootCAs() error {
 		return err
 	}
 	geoIndex := slices.Index(header, "Geographic Focus")
-	nameIndex := slices.Index(header, "Common Name or Certificate Name")
 	certIndex := slices.Index(header, "PEM Info")
 
-	generated := strings.Builder{}
-	generated.WriteString(`// Code generated by 'make update_certificates'. DO NOT EDIT.
-
-package certificate
-
-import "crypto/x509"
-
-var mozillaIncluded *x509.CertPool
-
-func init() {
-	mozillaIncluded = x509.NewCertPool()
-`)
+	pemBundle := strings.Builder{}
 	for {
 		record, err := reader.Read()
 		if err == io.EOF {
@@ -60,18 +49,12 @@ func init() {
 		if record[geoIndex] == "China" {
 			continue
 		}
-		generated.WriteString("\n	// ")
-		generated.WriteString(record[nameIndex])
-		generated.WriteString("\n")
-		generated.WriteString("	mozillaIncluded.AppendCertsFromPEM([]byte(`")
 		cert := record[certIndex]
-		// Remove single quotes
 		cert = cert[1 : len(cert)-1]
-		generated.WriteString(cert)
-		generated.WriteString("`))\n")
+		pemBundle.WriteString(cert)
+		pemBundle.WriteString("\n")
 	}
-	generated.WriteString("}\n")
-	return os.WriteFile("common/certificate/mozilla.go", []byte(generated.String()), 0o644)
+	return writeGeneratedCertificateBundle("mozilla", "mozillaIncluded", pemBundle.String())
 }
 
 func fetchChinaFingerprints() (map[string]bool, error) {
@@ -119,23 +102,11 @@ func updateChromeIncludedRootCAs() error {
 	if err != nil {
 		return err
 	}
-	subjectIndex := slices.Index(header, "Subject")
 	statusIndex := slices.Index(header, "Google Chrome Status")
 	certIndex := slices.Index(header, "X.509 Certificate (PEM)")
 	fingerprintIndex := slices.Index(header, "SHA-256 Fingerprint")
 
-	generated := strings.Builder{}
-	generated.WriteString(`// Code generated by 'make update_certificates'. DO NOT EDIT.
-
-package certificate
-
-import "crypto/x509"
-
-var chromeIncluded *x509.CertPool
-
-func init() {
-	chromeIncluded = x509.NewCertPool()
-`)
+	pemBundle := strings.Builder{}
 	for {
 		record, err := reader.Read()
 		if err == io.EOF {
@@ -149,18 +120,39 @@ func init() {
 		if chinaFingerprints[record[fingerprintIndex]] {
 			continue
 		}
-		generated.WriteString("\n	// ")
-		generated.WriteString(record[subjectIndex])
-		generated.WriteString("\n")
-		generated.WriteString("	chromeIncluded.AppendCertsFromPEM([]byte(`")
 		cert := record[certIndex]
-		// Remove single quotes if present
 		if len(cert) > 0 && cert[0] == '\'' {
 			cert = cert[1 : len(cert)-1]
 		}
-		generated.WriteString(cert)
-		generated.WriteString("`))\n")
+		pemBundle.WriteString(cert)
+		pemBundle.WriteString("\n")
+	}
+	return writeGeneratedCertificateBundle("chrome", "chromeIncluded", pemBundle.String())
+}
+
+func writeGeneratedCertificateBundle(name string, variableName string, pemBundle string) error {
+	goSource := `// Code generated by 'make update_certificates'. DO NOT EDIT.
+
+package certificate
+
+import (
+	"crypto/x509"
+	_ "embed"
+)
+
+//go:embed ` + name + `.pem
+var ` + variableName + `PEM string
+
+var ` + variableName + ` *x509.CertPool
+
+func init() {
+	` + variableName + ` = x509.NewCertPool()
+	` + variableName + `.AppendCertsFromPEM([]byte(` + variableName + `PEM))
+}
+`
+	err := os.WriteFile(filepath.Join("common/certificate", name+".pem"), []byte(pemBundle), 0o644)
+	if err != nil {
+		return err
 	}
-	generated.WriteString("}\n")
-	return os.WriteFile("common/certificate/chrome.go", []byte(generated.String()), 0o644)
+	return os.WriteFile(filepath.Join("common/certificate", name+".go"), []byte(goSource), 0o644)
 }