Optimize idle downloader for better performance

- Optimize firstMissing() using bitmap iterator instead of O(n) linear scan
  Now efficiently finds first gap in downloaded blocks by iterating
  through set bits and detecting where the sequence breaks

- Download multiple consecutive blocks per idle cycle (up to ~1 second)
  Previously downloaded only one 64KB block per second when idle
  Now streams consecutive blocks until time limit is reached

- Batch savePart() calls - save once at end of idle cycle instead of
  after every single block, reducing disk I/O significantly

- Reuse buffer across block downloads instead of allocating per block

- Add ingestDataBatch() for batch operations without immediate disk save

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: MagicalTux

README.md

@@ -214,4 +214,3 @@ On close:
 ## TODO
 
 - Add support for range invalidation (bad checksum causes re-download of affected area)
-- Refactor idle downloader for better performance

client.go

@@ -159,10 +159,9 @@ func (dl *dlClient) ReadAt(p []byte, off int64) (int, error) {
 }
 
 // idleTaskRun is called during idle periods to download missing blocks
-// in the background. It runs in a separate goroutine.
+// in the background. It runs in a separate goroutine and downloads multiple
+// consecutive blocks until approximately 1 second has elapsed.
 func (dl *dlClient) idleTaskRun() {
-	// this is run in a separate process
-
 	defer func() {
 		atomic.AddUintptr(&dl.taskCnt, ^uintptr(0))
 		atomic.AddUintptr(&dl.dlm.taskCnt, ^uintptr(0))
@@ -181,90 +180,89 @@ func (dl *dlClient) idleTaskRun() {
 	// increase timer now to avoid deletion
 	dl.expire = time.Now().Add(time.Minute)
 
-	if dl.reader != nil {
-		cnt := dl.handler.wantsFollowing(dl.rPos)
-		if cnt > 0 {
+	startTime := time.Now()
+	blocksDownloaded := 0
+	blkSize := dl.handler.getBlockSize()
+	buf := make([]byte, blkSize)
+
+	// Helper to save progress if we downloaded anything
+	defer func() {
+		if blocksDownloaded > 0 {
+			dl.handler.savePart()
+			dl.dlm.logf("idle: downloaded %d blocks in %v", blocksDownloaded, time.Since(startTime))
+		}
+	}()
+
+	// Download blocks until ~1 second has passed
+	for time.Since(startTime) < time.Second {
+		// Check if we have an existing reader at a useful position
+		if dl.reader != nil {
+			cnt := dl.handler.wantsFollowing(dl.rPos)
+			if cnt <= 0 {
+				// Current position already downloaded, close and find new position
+				dl.reader.Body.Close()
+				dl.reader = nil
+				continue
+			}
+
+			// Read the block
 			rPos := dl.rPos
-			// let's just read this from existing reader
-			buf := make([]byte, cnt)
-			n, err := io.ReadFull(dl.reader.Body, buf)
+			readBuf := buf[:cnt]
+			n, err := io.ReadFull(dl.reader.Body, readBuf)
 			if err != nil && err != io.ErrUnexpectedEOF {
 				dl.dlm.logf("idle read failed: %s", err)
 				dl.reader.Body.Close()
 				dl.reader = nil
+				if n == 0 {
+					continue
+				}
 			}
 			dl.rPos += int64(n)
 
-			// feed it
-			err = dl.handler.ingestData(buf[:n], rPos)
+			// Ingest without saving (we'll save once at the end)
+			err = dl.handler.ingestDataBatch(readBuf[:n], rPos)
 			if err != nil {
 				dl.dlm.logf("idle write failed: %s", err)
 				dl.failure = true
+				return
 			}
-			return
+			blocksDownloaded++
+			continue
 		}
 
-		dl.reader.Body.Close()
-		dl.reader = nil
-	}
-
-	// let's just ask where to start
-	off := dl.handler.firstMissing()
-	if off < 0 {
-		// do not download
-		if dl.handler.isComplete() {
-			dl.complete = true
+		// No reader, find first missing block
+		off := dl.handler.firstMissing()
+		if off < 0 {
+			if dl.handler.isComplete() {
+				dl.complete = true
+			}
+			return
 		}
-		return
-	}
-
-	// spawn a new reader
-	req, err := http.NewRequest("GET", dl.url, nil)
-	if err != nil {
-		dl.dlm.logf("idle: failed to create request: %s", err)
-		return
-	}
-
-	if off != 0 {
-		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", off))
-	}
-
-	dl.dlm.logf("idle: initializing HTTP connection download at byte %d~", off)
 
-	// should respond with code 206 Partial Content
-	resp, err := dl.dlm.Client.Do(req)
-	if err != nil {
-		dl.dlm.logf("idle download failed: %s", err)
-		return
-	}
-	if resp.StatusCode > 299 {
-		// that's bad
-		resp.Body.Close()
-		dl.dlm.logf("idle download failed due to status %s", resp.Status)
-		return
-	}
-	dl.reader = resp
-	dl.rPos = off
+		// Create new HTTP request
+		req, err := http.NewRequest("GET", dl.url, nil)
+		if err != nil {
+			dl.dlm.logf("idle: failed to create request: %s", err)
+			return
+		}
 
-	cnt := dl.handler.wantsFollowing(off)
-	if cnt <= 0 {
-		// why?
-		return
-	}
+		if off != 0 {
+			req.Header.Set("Range", fmt.Sprintf("bytes=%d-", off))
+		}
 
-	buf := make([]byte, cnt)
-	n, err := io.ReadFull(dl.reader.Body, buf)
-	if err != nil && err != io.ErrUnexpectedEOF {
-		dl.dlm.logf("idle read failed: %s", err)
-		dl.reader.Body.Close()
-		dl.reader = nil
-	}
-	dl.rPos += int64(n)
+		dl.dlm.logf("idle: initializing HTTP connection download at byte %d~", off)
 
-	// feed it (use separate thread to avoid deadlock)
-	err = dl.handler.ingestData(buf[:n], off)
-	if err != nil {
-		dl.dlm.logf("idle write failed: %s", err)
-		dl.failure = true
+		resp, err := dl.dlm.Client.Do(req)
+		if err != nil {
+			dl.dlm.logf("idle download failed: %s", err)
+			return
+		}
+		if resp.StatusCode > 299 {
+			resp.Body.Close()
+			dl.dlm.logf("idle download failed due to status %s", resp.Status)
+			return
+		}
+		dl.reader = resp
+		dl.rPos = off
 	}
 }

idle.go

@@ -67,26 +67,54 @@ func (f *File) firstMissing() int64 {
 		return -1 // can't be helped
 	}
 
-	// get number of blocks
 	blkCount := f.getBlockCount()
 
-	// find out first missing block
-	// TODO this can probably be optimized, roaring api may have something
-	for i := int64(0); i < blkCount; i++ {
-		if !f.status.Contains(uint32(i)) {
-			return i * f.blkSize
+	// If bitmap is empty, first missing is block 0
+	if f.status.IsEmpty() {
+		return 0
+	}
+
+	// Use iterator to efficiently find first gap in downloaded blocks.
+	// The iterator returns set bits in ascending order, so we look for
+	// the first position where the expected sequence breaks.
+	it := f.status.Iterator()
+	expected := uint32(0)
+
+	for it.HasNext() {
+		val := it.Next()
+		if val > expected {
+			// Found a gap - 'expected' is the first missing block
+			return int64(expected) * f.blkSize
 		}
+		expected = val + 1
+		if int64(expected) >= blkCount {
+			// We've covered all blocks we need
+			break
+		}
+	}
+
+	// Check if there are blocks after the last downloaded block
+	if int64(expected) < blkCount {
+		return int64(expected) * f.blkSize
 	}
 
-	// ?????
-	// did we have more blocks in status than we need? did file size change? this sounds like everything is likely corrupted...
 	return -1
 }
 
 // ingestData writes a block of data to the local file at the specified offset.
 // The offset must be block-aligned and the data must be exactly one block in size
-// (or the correct size for the final block).
+// (or the correct size for the final block). This method saves progress to disk.
 func (f *File) ingestData(b []byte, offset int64) error {
+	if err := f.ingestDataBatch(b, offset); err != nil {
+		return err
+	}
+	f.savePart()
+	return nil
+}
+
+// ingestDataBatch writes a block of data without saving progress to disk.
+// Use this for batch operations, then call savePart() once at the end.
+func (f *File) ingestDataBatch(b []byte, offset int64) error {
 	if !f.hasSize {
 		return errors.New("invalid operation, file size unknown")
 	}
@@ -125,11 +153,9 @@ func (f *File) ingestData(b []byte, offset int64) error {
 		return err
 	}
 
-	// mark blocks as received
+	// mark block as received
 	f.status.Add(uint32(block))
 
-	f.savePart()
-
 	return nil
 }
 

smartremote_test.go

@@ -44,15 +44,6 @@ func newTestServer() *httptest.Server {
 	}))
 }
 
-// newTestServerNoRange creates an HTTP server without Range support
-func newTestServerNoRange() *httptest.Server {
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Length", itoa(int64(len(testData))))
-		w.WriteHeader(http.StatusOK)
-		w.Write(testData)
-	}))
-}
-
 // newTestServerNoHead creates an HTTP server that rejects HEAD requests
 func newTestServerNoHead() *httptest.Server {
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {