WIP: Add httpchunksize option

Used to make youtube-dl do random range request around a specificed chunk size

Author: wader

go.mod

@@ -1,6 +1,6 @@
 module github.com/wader/ydls
 
-go 1.12
+go 1.13
 
 require (
 	// bump: leaktest /github.com\/fortytw2\/leaktest v(.*)/ git:https://github.com/fortytw2/leaktest.git|^1
@@ -19,3 +19,5 @@ require (
 	// bump: sync command go get -d golang.org/x/sync && go mod tidy
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 )
+
+replace github.com/wader/goutubedl => /Users/wader/src/goutubedl

internal/humnum/humnum.go

@@ -0,0 +1,41 @@
+package humnum
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+func Atoi(s string) (int, error) {
+	i := strings.IndexFunc(s, func(r rune) bool {
+		return !unicode.IsDigit(r)
+	})
+	if i == -1 {
+		i = len(s)
+	}
+
+	prefix, suffix := s[0:i], s[i:]
+	n, _ := strconv.Atoi(prefix)
+	m := 1
+	if suffix != "" {
+		switch strings.ToLower(suffix) {
+		case "k":
+			m = 1000
+		case "ki", "kibi":
+			m = 1024
+		case "m":
+			m = 1000 * 1000
+		case "mi", "mibi":
+			m = 1024 * 1024
+		case "g":
+			m = 1000 * 1000 * 1000
+		case "gi", "gibi":
+			m = 1024 * 1024 * 1024
+		default:
+			return 9, fmt.Errorf("unknown suffix %q", suffix)
+		}
+	}
+
+	return n * m, nil
+}

internal/humnum/numnum_test.go

@@ -0,0 +1,37 @@
+package humnum_test
+
+import (
+	"testing"
+
+	"github.com/wader/ydls/internal/humnum"
+)
+
+func TestAtoi(t *testing.T) {
+	testCases := []struct {
+		s           string
+		expectedN   int
+		expectedErr string
+	}{
+		{s: "", expectedN: 0},
+		{s: "1000", expectedN: 1000},
+		{s: "1K", expectedN: 1000},
+		{s: "1k", expectedN: 1000},
+		{s: "1M", expectedN: 1000_000},
+		{s: "1G", expectedN: 1000_000_000},
+		{s: "1Ki", expectedN: 1024},
+		{s: "1Mi", expectedN: 1024 * 1024},
+		{s: "1Gi", expectedN: 1024 * 1024 * 1024},
+		{s: "1Kibi", expectedN: 1024},
+		{s: "1Mibi", expectedN: 1024 * 1024},
+		{s: "1Gibi", expectedN: 1024 * 1024 * 1024},
+		{s: "123abc", expectedErr: `unknown suffix "abc"`},
+	}
+	for _, tC := range testCases {
+		t.Run(tC.s, func(t *testing.T) {
+			actualN, actualErr := humnum.Atoi(tC.s)
+			if (tC.expectedErr != "" && (actualErr == nil || tC.expectedErr != actualErr.Error())) && actualN != tC.expectedN {
+				t.Errorf("expected %v (%v), got %v (%v)", tC.expectedN, tC.expectedErr, actualN, actualErr)
+			}
+		})
+	}
+}

internal/ydls/requestoptions.go

@@ -6,17 +6,19 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/wader/ydls/internal/humnum"
 	"github.com/wader/ydls/internal/timerange"
 )
 
 // RequestOptions request options
 type RequestOptions struct {
-	MediaRawURL string              // youtubedl media URL
-	Format      *Format             // output format
-	Codecs      []string            // force codecs
-	Retranscode bool                // force retranscode even if same input codec
-	TimeRange   timerange.TimeRange // time range limit
-	Items       uint                // feed item count limit
+	MediaRawURL   string              // youtubedl media URL
+	Format        *Format             // output format
+	Codecs        []string            // force codecs
+	Retranscode   bool                // force retranscode even if same input codec
+	TimeRange     timerange.TimeRange // time range limit
+	Items         uint                // feed item count limit
+	HTTPChunkSize uint                // youtubedl http chunk size option
 }
 
 // NewRequestOptionsFromQuery /?url=...&format=...
@@ -70,13 +72,24 @@ func NewRequestOptionsFromQuery(v url.Values, formats Formats) (RequestOptions,
 		items = uint(itemsN)
 	}
 
+	httpChunkSize := uint(0)
+	httpChunkSizeStr := v.Get("httpchunksize")
+	if httpChunkSizeStr != "" {
+		httpChunkSizeN, httpChunkSizeNErr := humnum.Atoi(httpChunkSizeStr)
+		if httpChunkSizeNErr != nil {
+			return RequestOptions{}, fmt.Errorf("invalid HTTP chunk size")
+		}
+		httpChunkSize = uint(httpChunkSizeN)
+	}
+
 	return RequestOptions{
-		MediaRawURL: mediaRawURL,
-		Format:      format,
-		Codecs:      codecs,
-		Retranscode: v.Get("retranscode") != "",
-		TimeRange:   timeRange,
-		Items:       items,
+		MediaRawURL:   mediaRawURL,
+		Format:        format,
+		Codecs:        codecs,
+		Retranscode:   v.Get("retranscode") != "",
+		TimeRange:     timeRange,
+		Items:         items,
+		HTTPChunkSize: httpChunkSize,
 	}, nil
 }
 
@@ -173,7 +186,6 @@ func NewRequestOptionsFromOpts(opts []string, formats Formats) (RequestOptions,
 				return RequestOptions{}, fmt.Errorf("invalid items count")
 			}
 			r.Items = uint(itemsN)
-			strconv.ParseUint("", 10, 32)
 		} else if _, ok := codecNames[opt]; ok {
 			r.Codecs = append(r.Codecs, opt)
 		} else if tr, trErr := timerange.NewTimeRangeFromString(opt); trErr == nil {
@@ -206,5 +218,8 @@ func (r RequestOptions) QueryValues() url.Values {
 	if r.Items > 0 {
 		v.Set("items", strconv.Itoa(int(r.Items)))
 	}
+	if r.HTTPChunkSize != 0 {
+		v.Set("httpchunksize", strconv.Itoa(int(r.HTTPChunkSize)))
+	}
 	return v
 }

internal/ydls/ydls.go

@@ -427,6 +427,10 @@ func (ydls *YDLS) download(ctx context.Context, options DownloadOptions, attempt
 		}
 	}
 
+	if options.RequestOptions.HTTPChunkSize != 0 {
+		ydlOptions.HTTPChunkSize = options.RequestOptions.HTTPChunkSize
+	}
+
 	ydlResult, err := goutubedl.New(ctx, options.RequestOptions.MediaRawURL, ydlOptions)
 	if err != nil {
 		log.Printf("Failed to download: %s", err)