feat: consolidate .go-entry + .go-etag into bundle go.toml (RemoteSettings); always check freshness via ETag to source (per-startup file)

- New RemoteSettings + RemoteSettingsStore modeled after global Settings.
- go.toml at bundle root now holds 'entry' (for bare refs) and 'etags' map keyed by relative file path or :path.
- Remote refs always issue conditional GET (ETag); 304 keeps local, no more 2-week mtime gate for downloads (2w was only for cleanup).
- Cleanup now descends into host dirs to prune individual stale ref bundles by their root dir mtime (touched on use).
- Updated tests, docs, and resolver logic (GetStartupKey, etc.).
- Unrelated local changes left uncommitted.

Author: kzu

readme.md

@@ -84,12 +84,12 @@ The first argument is resolved by first checking if it is a local file (`File.Ex
 If not, it falls back to parsing it as a remote ref (`owner/repo[@ref][:path]`).
 
 Downloaded content is cached under the `dotnet/go` directory (same root as local apps)
-and participates in the normal up-to-date checks. By default, a remote ref is
-considered fresh for **2 weeks**. After that (or when using `--force` / `--go-force`),
-it will be re-downloaded.
+and participates in the normal up-to-date checks. Remote refs are always revalidated
+by sending a conditional request (using ETag when available) to the source. A 304
+Not Modified response means the local copy is used as-is.
 
 ```console
-# Force a fresh download even if within the 2-week window
+# Force a fresh download (bypasses ETag conditional check)
 dnx go --force kzu/sandbox
 
 # Same using the --go- alias form
@@ -124,9 +124,8 @@ dnx go clean app.cs
 dnx go clean --all
 ```
 
-Caches are also cleaned automatically: at most once every couple of days,
-`go#` removes cache directories that haven't been used for a while, in a
-detached background process. Apps you run regularly are never affected.
+Unused download locations and published binaries are periodically cleaned up
+in a detached background process. Apps you run regularly are never affected.
 
 ## Performance
 

src/Tests/EndToEndTests.cs

@@ -226,12 +226,12 @@ static string CreateTempDir()
     }
 
     [Fact]
-    public void Remote_stale_2week_re_download_refreshes_mtime_immediate_followup_does_not()
+    public void Remote_ref_always_revalidates_via_etag_root_dir_touched_source_files_not()
     {
         var scratch = @"C:\Users\kzu\AppData\Local\Temp\grok-goal-4376de8fe197\implementer";
         Directory.CreateDirectory(scratch);
         var remoteRef = "gist.github.com/kzu/0ac826dc7de666546aaedd38e5965381";
-        var logPath = Path.Combine(scratch, "stale-download.log");
+        var logPath = Path.Combine(scratch, "remote-etag.log");
 
         try
         {
@@ -243,7 +243,7 @@ public void Remote_stale_2week_re_download_refreshes_mtime_immediate_followup_do
             Assert.Equal(0, warm.ExitCode);
             Assert.Contains("run.cs", warm.Output);
 
-            // Locate the downloaded source cs for this ref (robust search, not brittle FirstOrDefault + loose contains)
+            // Locate the downloaded source cs and its bundle root dir for this ref
             var goRoot = GetTempRoot();
             var gistCandidates = Directory.GetDirectories(goRoot, "*0ac826dc7de666546aaedd38e5965381*", SearchOption.AllDirectories)
                 .OrderByDescending(d => d.Length)
@@ -256,36 +256,54 @@ public void Remote_stale_2week_re_download_refreshes_mtime_immediate_followup_do
             Assert.False(string.IsNullOrEmpty(sourceCs), "expected source .cs under download");
             var srcInfo = new FileInfo(sourceCs);
 
-            // Artificially age it >14 days
-            var staleTime = DateTime.UtcNow.AddDays(-20);
-            srcInfo.LastWriteTimeUtc = staleTime;
-            Assert.True((DateTime.UtcNow - srcInfo.LastWriteTimeUtc).TotalDays > 14);
+            // The bundle root (unzipped ref dir) is the gistRoot we found (contains the sources for this gist download).
+            var bundleRoot = gistRoot;
+            var rootInfo = new DirectoryInfo(bundleRoot);
+            var rootMtimeBefore = rootInfo.LastWriteTimeUtc;
 
-            // Re-invoke: should detect stale, (re)download/touch mtime to recent, succeed with marker
-            var (exitStale, outStale) = RunGo(remoteRef, "--", "-v:q");
-            File.WriteAllText(logPath, "STALE-RE-INVOKE:\n" + outStale);
-            Assert.Equal(0, exitStale);
-            Assert.Contains("run.cs", outStale);
+            // Second run (rapid): always performs ETag check; on 304 we touch *only* the root dir,
+            // individual source files are not touched.
+            var mtimeBefore = srcInfo.LastWriteTimeUtc;
+            var (exit2, out2) = RunGo(remoteRef, "--", "-v:q");
+            File.WriteAllText(logPath, "SECOND-RUN:\n" + out2);
+            Assert.Equal(0, exit2);
+            Assert.Contains("run.cs", out2);
 
             srcInfo.Refresh();
-            var afterStaleMtime = srcInfo.LastWriteTimeUtc;
-            File.AppendAllText(logPath, $"\nSOURCE-MTIME-AFTER-STALE: {afterStaleMtime:O}\n");
-            Assert.True((DateTime.UtcNow - afterStaleMtime).TotalMinutes < 5, "source mtime should be refreshed to recent by re-dl");
+            rootInfo.Refresh();
+            var mtimeAfter = srcInfo.LastWriteTimeUtc;
+            var rootMtimeAfter = rootInfo.LastWriteTimeUtc;
+
+            File.AppendAllText(logPath, $"\nSOURCE-MTIME-AFTER: {mtimeAfter:O}\nROOT-MTIME-AFTER: {rootMtimeAfter:O}\n");
+
+            // Source file mtime must not have been updated by resolver (no more per-file touch on 304 or extract).
+            var fileDelta = (mtimeAfter - mtimeBefore).TotalSeconds;
+            Assert.True(fileDelta < 2.0, $"source file mtime must be unchanged on revalidation (delta={fileDelta}s)");
 
-            // Immediate follow-up: must NOT update the source mtime (plan requirement)
-            var mtimeBeforeImmediate = srcInfo.LastWriteTimeUtc;
+            // Root download dir *must* be touched to mark usage for cleanup.
+            Assert.True(rootMtimeAfter >= rootMtimeBefore, "bundle root dir mtime should be updated on use");
+
+            // Immediate third run: same, no source mtime change.
+            var mtimeBeforeImm = srcInfo.LastWriteTimeUtc;
+            var rootBeforeImm = rootInfo.LastWriteTimeUtc;
             var (exitImm, outImm) = RunGo(remoteRef, "--", "-v:q");
-            File.AppendAllText(logPath, "IMMEDIATE-FOLLOWUP:\n" + outImm);
+            File.AppendAllText(logPath, "IMMEDIATE:\n" + outImm);
             Assert.Equal(0, exitImm);
-            Assert.Contains("run.cs", outImm);
 
             srcInfo.Refresh();
-            var mtimeAfterImmediate = srcInfo.LastWriteTimeUtc;
-            File.AppendAllText(logPath, $"SOURCE-MTIME-AFTER-IMMEDIATE: {mtimeAfterImmediate:O}\n");
-
-            var deltaSeconds = (mtimeAfterImmediate - mtimeBeforeImmediate).TotalSeconds;
-            // The immediate run (cache hit, no dl) must not have updated the source mtime via our logic or observable change.
-            Assert.True(deltaSeconds < 2.0, $"immediate follow-up must not update source mtime (delta={deltaSeconds}s)");
+            rootInfo.Refresh();
+            var mtimeAfterImm = srcInfo.LastWriteTimeUtc;
+            var rootAfterImm = rootInfo.LastWriteTimeUtc;
+            File.AppendAllText(logPath, $"SOURCE-MTIME-IMM: {mtimeAfterImm:O}\nROOT-MTIME-IMM: {rootAfterImm:O}\n");
+
+            Assert.True((mtimeAfterImm - mtimeBeforeImm).TotalSeconds < 2.0, "immediate must not change source file mtime");
+            Assert.True(rootAfterImm > rootBeforeImm || rootAfterImm >= rootBeforeImm, "root dir touched on immediate use too");
+
+            // --force should succeed (forces full fetch, bypassing any conditional).
+            var (exitForce, outForce) = RunGo("--force", remoteRef, "--", "-v:q");
+            File.AppendAllText(logPath, "FORCE:\n" + outForce);
+            Assert.Equal(0, exitForce);
+            Assert.Contains("run.cs", outForce);
         }
         finally
         {

src/Tests/GoBuildCacheTests.cs

@@ -135,47 +135,49 @@ public void GetRemoteEntryPointPath_uses_path_or_program_or_persisted_marker()
         p = Devlooped.RemoteSourceResolver.GetRemoteEntryPointPath(withPath);
         Assert.EndsWith("src/app.cs", p.Replace('\\', '/'));
 
-        // simulate persist
+        // simulate persist via consolidated go.toml
         var dir = Devlooped.RemoteSourceResolver.GetRemoteEntryPointPath(remoteNoPath);
         var tempDir = Path.GetDirectoryName(dir)!;
         Directory.CreateDirectory(tempDir);
-        var marker = Path.Combine(tempDir, ".go-entry");
-        File.WriteAllText(marker, "actual.cs");
+        var settings = new Devlooped.RemoteSettings { Entry = "actual.cs" };
+        Devlooped.RemoteSettingsStore.Save(settings, tempDir);
         p = Devlooped.RemoteSourceResolver.GetRemoteEntryPointPath(remoteNoPath);
         Assert.EndsWith("actual.cs", p);
 
-        // cleanup marker for hygiene
-        try { File.Delete(marker); } catch { }
+        // cleanup for hygiene
+        try { File.Delete(Devlooped.RemoteSettingsStore.GetFilePath(tempDir)); } catch { }
 
-        // NEW: exercise dir-exists + no-marker discovery branch (must prefer program.cs deterministically, write marker)
+        // NEW: exercise dir-exists + no-entry discovery branch (must prefer program.cs deterministically, write to go.toml)
         var uniq2 = Guid.NewGuid().ToString("N")[..8];
         var remoteDisc = new Devlooped.RemoteRef("disc" + uniq2, "repo" + uniq2, null, null, null);
         var cand = Devlooped.RemoteSourceResolver.GetRemoteEntryPointPath(remoteDisc);
         var discDir = Path.GetDirectoryName(cand)!;
         Directory.CreateDirectory(discDir);
-        var m2 = Path.Combine(discDir, ".go-entry");
-        if (File.Exists(m2)) File.Delete(m2);
+        var settingsPath = Devlooped.RemoteSettingsStore.GetFilePath(discDir);
+        if (File.Exists(settingsPath)) File.Delete(settingsPath);
 
         // write program.cs + another to test prefer
         File.WriteAllText(Path.Combine(discDir, "other.cs"), "// other");
         var prog = Path.Combine(discDir, "program.cs");
         File.WriteAllText(prog, "// prog");
         var discovered = Devlooped.RemoteSourceResolver.GetRemoteEntryPointPath(remoteDisc);
         Assert.Equal(prog, discovered);
-        Assert.True(File.Exists(m2));
-        Assert.Equal("program.cs", File.ReadAllText(m2).Trim());
+        Assert.True(File.Exists(settingsPath));
+        var s1 = Devlooped.RemoteSettingsStore.Load(discDir);
+        Assert.Equal("program.cs", s1.Entry);
 
         // case with no program.cs: should pick the (only) remaining .cs
         File.Delete(prog);
         var other = Path.Combine(discDir, "other.cs");
         if (File.Exists(other)) File.Delete(other);
-        if (File.Exists(m2)) File.Delete(m2);
+        if (File.Exists(settingsPath)) File.Delete(settingsPath);
         File.WriteAllText(Path.Combine(discDir, "only.cs"), "// only");
         var onlyDisc = Devlooped.RemoteSourceResolver.GetRemoteEntryPointPath(remoteDisc);
         Assert.EndsWith("only.cs", onlyDisc);
-        Assert.Equal("only.cs", File.ReadAllText(m2).Trim());
+        var s2 = Devlooped.RemoteSettingsStore.Load(discDir);
+        Assert.Equal("only.cs", s2.Entry);
 
-        try { File.Delete(m2); Directory.Delete(discDir, true); } catch { }
+        try { File.Delete(settingsPath); Directory.Delete(discDir, true); } catch { }
     }
 
     [Fact]

src/go/CleanupManager.cs

@@ -18,6 +18,22 @@ public static void CleanupStaleDirectories(string root, int days)
 
         foreach (var directory in Directory.EnumerateDirectories(root))
         {
+            var name = Path.GetFileName(directory);
+            if (name.Contains('.'))
+            {
+                // Remote host container (e.g. github.com, gist.github.com). Descend to clean
+                // individual ref root (unzipped download) directories by their mtime.
+                CleanupRemoteDownloadLocations(directory, cutoff);
+                // Prune empty host container if all refs under it were removed.
+                try
+                {
+                    if (!Directory.EnumerateFileSystemEntries(directory).Any())
+                        Directory.Delete(directory);
+                }
+                catch { }
+                continue;
+            }
+
             if (Directory.GetLastWriteTimeUtc(directory) >= cutoff)
                 continue;
 
@@ -31,4 +47,50 @@ public static void CleanupStaleDirectories(string root, int days)
             }
         }
     }
+
+    static void CleanupRemoteDownloadLocations(string hostDir, DateTime cutoff)
+    {
+        // Find leaf-ish ref bundle roots: directories containing top-level .cs files or a go.toml
+        // (consolidated settings for entry + etag). We only consider their own LastWriteTimeUtc
+        // (touched on use of any file in the bundle).
+        foreach (var dir in EnumeratePotentialBundleDirs(hostDir))
+        {
+            try
+            {
+                if (!Directory.Exists(dir))
+                    continue;
+                if (Directory.GetLastWriteTimeUtc(dir) >= cutoff)
+                    continue;
+
+                // Confirm it looks like one of our download bundles
+                if (Directory.EnumerateFiles(dir, "*.cs", SearchOption.TopDirectoryOnly).Any() ||
+                    File.Exists(Path.Combine(dir, "go.toml")))
+                {
+                    Directory.Delete(dir, recursive: true);
+                }
+            }
+            catch
+            {
+                // Best-effort per location.
+            }
+        }
+    }
+
+    static IEnumerable<string> EnumeratePotentialBundleDirs(string start)
+    {
+        var stack = new Stack<string>();
+        stack.Push(start);
+        while (stack.Count > 0)
+        {
+            var current = stack.Pop();
+            // Yield current if it may be a bundle root (we'll filter by contents at call site).
+            yield return current;
+            try
+            {
+                foreach (var sub in Directory.EnumerateDirectories(current))
+                    stack.Push(sub);
+            }
+            catch { }
+        }
+    }
 }

src/go/RemoteSettings.cs

@@ -0,0 +1,81 @@
+using System.Text;
+using System.Text.Json.Serialization;
+using Tomlyn;
+using Tomlyn.Serialization;
+
+namespace Devlooped;
+
+public class RemoteSettings
+{
+    public string? Entry { get; set; }
+
+    /// <summary>
+    /// ETags keyed by the relative file path within the bundle (e.g. "program.cs" or "src/hello.cs")
+    /// or equivalently the path portion from a remote ref (owner/repo:path).
+    /// The go.toml at the bundle root holds ETags for all such refs/entries within the same bundle.
+    /// </summary>
+    public Dictionary<string, string>? ETags { get; set; }
+}
+
+public static class RemoteSettingsStore
+{
+    public static string GetFilePath(string directory)
+        => Path.Combine(directory, "go.toml");
+
+    public static RemoteSettings Load(string? path = null)
+    {
+        if (string.IsNullOrEmpty(path))
+            return new RemoteSettings();
+
+        path = ResolveTomlPath(path);
+
+        if (!File.Exists(path))
+            return new RemoteSettings();
+
+        var text = File.ReadAllText(path);
+        return TomlSerializer.Deserialize(text, TomlContext.Default.RemoteSettings) ?? new RemoteSettings();
+    }
+
+    public static void Save(RemoteSettings settings, string? path = null)
+    {
+        if (string.IsNullOrEmpty(path))
+            path = "go.toml";
+
+        path = ResolveTomlPath(path);
+
+        var dir = Path.GetDirectoryName(path);
+        if (!string.IsNullOrEmpty(dir))
+            Directory.CreateUserDirectory(dir);
+
+        var text = TomlSerializer.Serialize(settings, TomlContext.Default.RemoteSettings);
+        File.WriteAllText(path, text, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
+    }
+
+    public static string? GetETag(RemoteSettings settings, string key)
+    {
+        if (settings.ETags == null) return null;
+        if (settings.ETags.TryGetValue(key, out var value)) return value;
+        // Case-insensitive fallback for robustness across platforms/casing in refs
+        foreach (var kv in settings.ETags)
+            if (string.Equals(kv.Key, key, StringComparison.OrdinalIgnoreCase))
+                return kv.Value;
+        return null;
+    }
+
+    public static void SetETag(RemoteSettings settings, string key, string etag)
+    {
+        settings.ETags ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        settings.ETags[key] = etag;
+    }
+
+    static string ResolveTomlPath(string path)
+    {
+        // Treat as directory if it exists as a directory or doesn't look like a .toml file path
+        if (Directory.Exists(path) || !path.EndsWith(".toml", StringComparison.OrdinalIgnoreCase))
+            return Path.Combine(path, "go.toml");
+        return path;
+    }
+}
+
+[TomlSerializable(typeof(RemoteSettings))]
+partial class TomlContext : TomlSerializerContext { }