FIX: Remote Source lists (in UI and Launcher) are empty under some other circumstances

Author: Nick Daniels

Assets/TestProto.js

@@ -1 +1 @@
-using Microsoft.Extensions.Logging;using Microsoft.Extensions.Options;using System.Net;using System.Reactive.Linq;namespace Gry.Caching;public partial class HttpCache{    public string? CachePath(string uri)    {        var safeName = SafeFileName(uri);        var path = Path.Join(CacheFolder, safeName);        if (File.Exists(path))        {            return path;        }        return null;    }    public (string, string) CacheFileNames(string uri)    {        var safeName = SafeFileName(uri);        var path = Path.Join(CacheFolder, safeName);        var name = Path.GetFileName(uri);        return (path, name);    }    private async Task CheckConnection()    {        try        {            var entry = await Dns.GetHostEntryAsync("raw.githubusercontent.com");            InternetAvailable = entry.AddressList.Length > 0;        }        catch        {            Logger.LogDebug("No Internet Connection");            InternetAvailable = false;        }    }    public async Task<HttpResponseMessage?> Head(string uri)    {        try        {            return await Http.SendAsync(new(HttpMethod.Head, uri));        }        catch (Exception ex)        {            Logger.LogWarning("Error making HEAD request: {}", ex.Message);            return new HttpResponseMessage(HttpStatusCode.BadRequest);        }    }    public async Task<PathStatus> GetPathStatus(string uri, string? path = null)    {        path ??= Path.Join(CacheFolder, SafeFileName(uri));        var pathExists = File.Exists(path);        var lastCached = Cache.LastChange(path);        if (pathExists &&            lastCached.AddMinutes(Settings.MinimumCacheTimeMinutes) > DateTime.Now        )        {            return new(false, true, true, lastCached, HttpStatusCode.OK);        }        var head = await Head(uri);        if (!pathExists && head?.IsSuccessStatusCode is false)        {            return new(false, false, false, DateTime.MinValue, head?.StatusCode, head?.ReasonPhrase); //No download, not found, no local        }        var length = 0;        if (head?.Headers.TryGetValues("Content-Length", out var lengths) is true)            length = int.Parse(lengths.First());        if (!pathExists)        {            return new(true, true, false, DateTime.MinValue, head?.StatusCode, Length: length); //Download, Found, None        }        var modified = head?.Content.Headers.LastModified?.LocalDateTime;        modified ??= lastCached > DateTime.MinValue ?                        lastCached.AddMinutes(Settings.MinimumCacheTimeMinutes) :                        DateTime.MinValue;        if (modified > lastCached)        {            return new(true, true, true, lastCached, head?.StatusCode ?? HttpStatusCode.Found, Length: length);        }        return new(false, true, true, lastCached, head?.StatusCode ?? HttpStatusCode.NotModified, Length: length);    }}
+using Microsoft.Extensions.Logging;using Microsoft.Extensions.Options;using System.Net;using System.Reactive.Linq;namespace Gry.Caching;public partial class HttpCache{    public string? CachePath(string uri)    {        var safeName = SafeFileName(uri);        var path = Path.Join(CacheFolder, safeName);        if (File.Exists(path))        {            return path;        }        return null;    }    public (string, string) CacheFileNames(string uri)    {        var safeName = SafeFileName(uri);        var path = Path.Join(CacheFolder, safeName);        var name = Path.GetFileName(uri);        return (path, name);    }    private async Task CheckConnection()    {        try        {            var entry = await Dns.GetHostEntryAsync("raw.githubusercontent.com");            InternetAvailable = entry.AddressList.Length > 0;        }        catch        {            Logger.LogDebug("No Internet Connection");            InternetAvailable = false;        }    }    public async Task<HttpResponseMessage?> Head(string uri)    {        try        {            return await Http.SendAsync(new(HttpMethod.Head, uri));        }        catch (Exception ex)        {            Logger.LogWarning("Error making HEAD request: {}", ex.Message);            return new HttpResponseMessage(HttpStatusCode.BadRequest);        }    }    public async Task<PathStatus> GetPathStatus(string uri, string? path = null)    {        path ??= Path.Join(CacheFolder, SafeFileName(uri));        var pathExists = File.Exists(path);        var lastCached = Cache.LastChange(path);        if (pathExists &&            lastCached.AddMinutes(Settings.MinimumCacheTimeMinutes) > DateTime.Now        )        {            return new(false, true, true, lastCached, HttpStatusCode.OK);        }        var head = await Head(uri);        if (head?.IsSuccessStatusCode is false && !pathExists)            return new(false, false, false, DateTime.MinValue, head?.StatusCode, head?.ReasonPhrase); //No download, not found, no local                var length = 0;        if (head?.Headers.TryGetValues("Content-Length", out var lengths) is true)            length = int.Parse(lengths.First());        if (!pathExists)            return new(true, true, false, DateTime.MinValue, head?.StatusCode, Length: length); //Download, Found, None                var modified = head?.Content.Headers.LastModified?.LocalDateTime;        modified ??= lastCached > DateTime.MinValue ?                        lastCached.AddMinutes(Settings.MinimumCacheTimeMinutes) :                        DateTime.MinValue;        if (modified > lastCached)            return new(true, true, true, modified.Value, head?.StatusCode ?? HttpStatusCode.Found, Length: length);                return new(false, true, true, lastCached, head?.StatusCode ?? HttpStatusCode.NotModified, Length: length);    }}

Assets/TestProto.py

@@ -7,10 +7,10 @@
     public static class Emulator
     {
         public const string Branch = "release";
-        public const int Build = 2;
+        public const int Build = 3;
         public const int Major = 1;
         public const int Minor = 3;
-        public const int Release = 3;
+        public const int Release = 4;
         public static readonly string Id = $"NABU NetSim v{Version}";
         public static readonly string Version = $"{Major}.{Minor}.{Build}-{Branch}{Release}";
     }

Dependencies/Gry.Caching/HttpCache.Internal.cs

@@ -0,0 +1 @@
+using Nabu.Settings;using Nabu.Sources;using Newtonsoft.Json.Linq;using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Threading.Tasks;using System.Xml;namespace Nabu.Network;public partial class NabuNetwork{    protected static bool IsEncryptedPak(string path) => path.EndsWith(Constants.EncryptedPakExtension);    protected static bool IsNabu(string path) => path.EndsWith(Constants.NabuExtension);    protected static bool IsPak(string path) => path.EndsWith(Constants.PakExtension);    protected static bool IsRawPak(string path) => PakFile().IsMatch(path);    protected static bool IsWebSource(string path) => path.Contains("://");    public ProgramSource? Source(AdaptorSettings settings)        => Source(settings.Source);    public ProgramSource? Source(string? name)        => Sources.Get(name);    public IEnumerable<NabuProgram> Programs(AdaptorSettings settings)    {        var source = Source(settings);        if (source is null)            return Array.Empty<NabuProgram>();        return Programs(source);    }    public IEnumerable<NabuProgram> Programs(string? name)    {        var source = Source(name);        if (source is null)            return Array.Empty<NabuProgram>();        return Programs(source);    }    public IEnumerable<NabuProgram> Programs(ProgramSource? source)    {        if (source is null)            return [];        if (SourceCache.TryGetValue(source.Name, out IEnumerable<NabuProgram>? value))            return value;        return Array.Empty<NabuProgram>();    }    protected (bool, string, ImageType) ContainsPak(string[] files)    {        var encryptedMenuPakName = NabuLib.PakName(Constants.CycleMenuNumber);        foreach (var f in files)        {            var filename = Path.GetFileNameWithoutExtension(f);            if (filename is Constants.CycleMenuPak)            {                var (supported, type) = IsSupportedType(f);                if (!supported) continue;                return (supported, f, type);            }            if (filename == encryptedMenuPakName)            {                var (supported, type) = IsSupportedType(f);                if (!supported) continue;                return (supported, f, type);            }        }        return (false, string.Empty, ImageType.None);    }    protected async Task<(bool, NabuProgram[])> IsKnownListType(ProgramSource source)    {        var tasks = new Task<(bool, NabuProgram[])>[] {            IsNabuCaJson(source),            IsNabuNetworkList(source)        };        foreach (var task in tasks)        {            var (isList, programs) = await task;            if (isList) return (isList, programs);        }                return (false, Array.Empty<NabuProgram>());    }    protected (bool, ImageType) IsSupportedType(string file)    {        var ext = Path.GetExtension(file);        return ext switch        {            _ when IsRawPak(file) => (true, ImageType.Raw),            Constants.PakExtension => (true, ImageType.Pak),            Constants.EncryptedPakExtension => (true, ImageType.EncryptedPak),            _ => (false, ImageType.None)        };    }    protected async Task<(bool, NabuProgram[])> IsNabuCaJson(ProgramSource source)    {        var uri = source.Path;        if (!(uri.EndsWith(".json") && uri.Contains("nabu.ca"))) { return (false, Array.Empty<NabuProgram>()); }        var (_, found, cached, _, _, _, _) = await Http.GetPathStatus(uri);        if (!found && !cached)        {            Warning($"Could not find {uri}, and it is not cached");            return (false, []);        }        try        {            var progs = new List<NabuProgram>();            JToken? sections = null;            while (sections == null)                try                {                    var json = await Http.GetString(uri);                    sections = JObject.Parse(json)["Items"];                }                catch (Exception ex)                {                    Warning(ex.Message);                }            string[] skipSections = ["NABU Cycles", "Demos", "Utilities"];            foreach (var section in sections)            {                var title = section["Title"]?.ToString();                if (skipSections.Contains(title)) continue;                var files = section["Files"];                if (files is null) continue;                foreach (var item in files)                {                    var name = item["Title"]?.ToString() ?? "Unnamed Program";                    var filename = item["Filename"]?.ToString() ?? Empty;                    var url = $"https://cloud.nabu.ca/HomeBrew/titles/{filename}";                    var author = item["Author"]?.ToString() ?? Empty;                    var description = item["Description"]?.ToString() ?? Empty;                    var tileColor = item["IconTileColor"]?.ToString() ?? CommonUI.BlankIconClrStr;                    var tilePattern = item["IconTilePattern"]?.ToString() ?? CommonUI.BlankIconPtrnStr;                    var headless = item["IsHeadless"]?.ToObject<bool>() is true;                    if (headless && !source.HeadlessMenu)                        source.HeadlessMenu = true;                    progs.Add(new(                        name,                        filename,                        source.Name,                        url,                        SourceType.Remote,                        ImageType.Raw,                        DefaultPatches,                        author,                        description,                        tileColor,                        tilePattern,                        category: title,                        headless: headless                    ));                }            }            return (true, progs.ToArray());        }        catch        {            return (false, []);        }    }    protected async Task<(bool, NabuProgram[])> IsNabuNetworkList(ProgramSource source)    {        var uri = source.Path;        if (!uri.EndsWith(".xml") && !uri.Contains("nabunetwork.com"))        {            return (false, Array.Empty<NabuProgram>());        }        var (_, found, cached, _, _, _, _) = await Http.GetPathStatus(uri);        if (!found && !cached)            return (false, []);        try        {            var xml = await Http.GetString(uri);            var list = new XmlDocument();            list.LoadXml(xml);            var cycleNodes = list.DocumentElement?.SelectNodes("Target");            if (cycleNodes is null) { return (false, Array.Empty<NabuProgram>()); }            var progs = new List<NabuProgram>();            foreach (XmlNode cycleNode in cycleNodes)            {                var targetType = cycleNode["TargetType"]?.InnerText;                if (targetType == "NabuNetwork")                    continue;                var name = cycleNode["Name"]?.InnerText ?? "Unnamed Program";                var url = cycleNode["Url"]?.InnerText ?? Empty;                url = url.Replace("loader.nabu", "000002.nabu");                progs.Add(new(                    name,                    name,                    source.Name,                    url,                    SourceType.Remote,                    ImageType.Raw,                    DefaultPatches,                    source.Author ?? Empty,                    Empty,                    CommonUI.BlankIconClrStr,                    CommonUI.BlankIconPtrnStr,                    category: targetType                ));            }            return (true, progs.ToArray());        }        catch        {            return (false, []);        }    }    protected async Task<(bool, string, ImageType)> IsPak(string url, int pak)    {        url = url.TrimEnd('/');        var type = url switch        {            _ when IsRawPak(url) => ImageType.Raw,            _ when IsPak(url) => ImageType.Pak,            _ when IsEncryptedPak(url) => ImageType.EncryptedPak,            _ => ImageType.None        };        bool found, cached;        if (type is ImageType.None)        {            (_, found, cached, _, _, _, _) = await Http.GetPathStatus($"{url}/{FormatTriple(pak)}{Constants.NabuExtension}");            if (found || cached)                return (true, url, ImageType.Raw);            (_, found, cached, _, _, _, _) = await Http.GetPathStatus($"{url}/{FormatTriple(pak)}{Constants.PakExtension}");            if (found || cached)                return (true, url, ImageType.Pak);            (_, found, cached, _, _, _, _) = await Http.GetPathStatus($"{url}/{NabuLib.PakName(pak)}{Constants.PakExtension}");            if (found || cached)                return (true, url, ImageType.Pak);            (_, found, cached, _, _, _, _) = await Http.GetPathStatus($"{url}/{NabuLib.PakName(pak)}{Constants.EncryptedPakExtension}");            if (found || cached)                return (false, url, ImageType.EncryptedPak); //Encrypted pak support is disabled.                                                   return (false, url, ImageType.None);        }        (_, found, cached, _, _, _, _) = await Http.GetPathStatus(url);        return (found || cached, url, type);    }}