feat: sftp (#51)

* feat: sftp
test: add tests for ftp and sftp

* chore: ci fixes

* chore: potential race fix

* fix: simplify existence checks

* fix: split path differently for ftp

* fix: 🤷

* chore: add debug print

* chore: lint

* chore: idk dude

* chore: ?

* chore: more logs

* chore: wipe mods before tests

* chore: logs

* chore: wat

* chore: wait?

* chore: no errors

* chore: gh actions are 💩

* fix: always sync after copy

* chore: remove some test logs

* chore: remove test progress watcher

* refactor: change progress to writer

* chore: logs

* chore: different logs

* chore: whoops

* chore: moar logs

* chore: even moar logs

* chore: what is life

* chore: why are we here

* chore: we are just bags of water floating through space

* chore: are you real?

* chore: ?

* chore: if you get a single update now I call bs

* chore: ok what if we just do one?

* chore: ok what if we do two?

* chore: this should not work

* chore: wait no, this one

* chore: fml

* chore: remove logs

* chore: what if we just wait a little

* chore: retry

* chore: move error

* chore: verbose log

* chore: remove explicit sleep

* chore: remove debug

* fix: linux pathing on windows

* fix: clean paths properly

* fix: fuck ftp

* fix: send update on vanilla

* feat: parallel ftp

* fix: remove potential credential leak

Author: Vilsol

.github/workflows/push.yaml

@@ -82,6 +82,10 @@ jobs:
       - name: Install Satisfactory Dedicated Server
         run: steamcmd +login anonymous +force_install_dir ${{ github.workspace }}/SatisfactoryDedicatedServer +app_update 1690800 validate +quit
 
+      - name: Change directory permissions
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: mkdir -p ${{ github.workspace }}/SatisfactoryDedicatedServer/FactoryGame/Mods && chmod -R 777 ${{ github.workspace }}/SatisfactoryDedicatedServer
+
       - name: List directory (linux)
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: ls -lR
@@ -90,6 +94,10 @@ jobs:
         if: ${{ matrix.os == 'windows-latest' }}
         run: tree /F
 
+      - name: Boot ftp and sftp
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: docker-compose -f docker-compose-test.yml up -d
+
       - name: Download GQL schema
         run: "npx graphqurl https://api.ficsit.dev/v2/query --introspect -H 'content-type: application/json' > schema.graphql"
 

.gitignore

@@ -128,4 +128,5 @@ dist/
 /.graphqlconfig
 schema.graphql
 *.log
-.direnv
+.direnv
+/SatisfactoryDedicatedServer

cfg/test_defaults.go

@@ -1,6 +1,8 @@
 package cfg
 
 import (
+	"log/slog"
+	"os"
 	"path/filepath"
 	"runtime"
 
@@ -18,4 +20,8 @@ func SetDefaults() {
 	viper.SetDefault("api-base", "https://api.ficsit.dev")
 	viper.SetDefault("graphql-api", "/v2/query")
 	viper.SetDefault("concurrent-downloads", 5)
+
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	})))
 }

cli/cache/download.go

@@ -4,11 +4,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
+	"github.com/avast/retry-go"
 	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/spf13/viper"
 
@@ -84,7 +87,11 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- ut
 	outer:
 		for {
 			select {
-			case update := <-upstreamUpdates:
+			case update, ok := <-upstreamUpdates:
+				if !ok {
+					break outer
+				}
+
 				for _, u := range group.updates {
 					u <- update
 				}
@@ -94,11 +101,29 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- ut
 		}
 	}()
 
-	size, err := downloadInternal(cacheKey, location, hash, url, upstreamUpdates, downloadSemaphore)
+	var size int64
+
+	err := retry.Do(func() error {
+		var err error
+		size, err = downloadInternal(cacheKey, location, hash, url, upstreamUpdates, downloadSemaphore)
+		if err != nil {
+			return fmt.Errorf("internal download error: %w", err)
+		}
+		return nil
+	},
+		retry.Attempts(5),
+		retry.Delay(time.Second),
+		retry.DelayType(retry.FixedDelay),
+		retry.OnRetry(func(n uint, err error) {
+			if n > 0 {
+				slog.Info("retrying download", slog.Uint64("n", uint64(n)), slog.String("cacheKey", cacheKey))
+			}
+		}),
+	)
 	if err != nil {
 		group.err = err
 		close(group.wait)
-		return nil, 0, err
+		return nil, 0, err // nolint
 	}
 
 	close(upstreamWaiter)
@@ -175,16 +200,17 @@ func downloadInternal(cacheKey string, location string, hash string, url string,
 	}
 
 	progresser := &utils.Progresser{
-		Reader:  resp.Body,
 		Total:   resp.ContentLength,
 		Updates: updates,
 	}
 
-	_, err = io.Copy(out, progresser)
+	_, err = io.Copy(io.MultiWriter(out, progresser), resp.Body)
 	if err != nil {
 		return 0, fmt.Errorf("failed writing file to disk: %w", err)
 	}
 
+	_ = out.Sync()
+
 	if updates != nil {
 		updates <- utils.GenericProgress{Completed: resp.ContentLength, Total: resp.ContentLength}
 	}

cli/context.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"fmt"
+	"log/slog"
 
 	"github.com/Khan/genqlient/graphql"
 	"github.com/spf13/viper"
@@ -87,6 +88,8 @@ func (g *GlobalContext) ReInit() error {
 
 // Wipe will remove any trace of ficsit anywhere
 func (g *GlobalContext) Wipe() error {
+	slog.Info("wiping global context")
+
 	// Wipe all installations
 	for _, installation := range g.Installations.Installations {
 		if err := installation.Wipe(); err != nil {