diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj new file mode 100644 index 0000000..af3792a --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj @@ -0,0 +1,31 @@ + + + + net9.0 + enable + enable + + false + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + PreserveNewest + + + + diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs new file mode 100644 index 0000000..2fd7f06 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs @@ -0,0 +1,23 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Tests +{ + [TestClass] + public sealed class EpicManifestParserTests + { + [TestMethod] + public async Task DoesItWork() + { + // Arrange + var manifestBuffer = await File.ReadAllBytesAsync(Path.Combine("TestFiles", "1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest")); + + // Act + var manifest = EpicManifestParser.Deserialize(manifestBuffer); + + // Assert + Assert.AreEqual("Super Space Club.exe", manifest.Meta.LaunchExe); + Assert.AreEqual("eccf14e21df84712a54d2b89b20d15f9", manifest.Meta.AppName); + Assert.AreEqual("9lEUV1B3tU-lv8teGLbqaQ", manifest.Meta.BuildId); + } + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/TestFiles/1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/TestFiles/1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest new file mode 100644 index 0000000..efd0bbb Binary files /dev/null and b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/TestFiles/1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest differ diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Api/ManifestInfo.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Api/ManifestInfo.cs new file mode 100644 index 0000000..8f3db2b --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Api/ManifestInfo.cs @@ -0,0 +1,305 @@ +using DeveLanCacheUI_Backend.EpicManifestParser.UE; +using Flurl; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Api; +// ReSharper disable UseSymbolAlias + +/// +public class ManifestInfo +{ + /// + public required List Elements { get; set; } + + /// + /// Parses the UTF-8 encoded text representing a single JSON value into a . + /// + /// A representation of the JSON value. + /// JSON text to parse. + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the buffer. + /// + public static ManifestInfo? Deserialize(ReadOnlySpan utf8Json) + => JsonSerializer.Deserialize(utf8Json, EpicManifestParserJsonContext.Default.ManifestInfo); + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON data to parse. + /// + /// is . + /// + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + public static ManifestInfo? Deserialize(Stream utf8Json) + => JsonSerializer.Deserialize(utf8Json, EpicManifestParserJsonContext.Default.ManifestInfo); + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON data to parse. + /// + /// The that can be used to cancel the read operation. + /// + /// + /// is . + /// + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + public static ValueTask DeserializeAsync(Stream utf8Json, CancellationToken cancellationToken = default) + => JsonSerializer.DeserializeAsync(utf8Json, EpicManifestParserJsonContext.Default.ManifestInfo, cancellationToken); + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON file to parse. + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + /// + public static ManifestInfo? DeserializeFile(string path) + { + using var fs = File.OpenRead(path); + return JsonSerializer.Deserialize(fs, EpicManifestParserJsonContext.Default.ManifestInfo); + } + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON file to parse. + /// + /// The that can be used to cancel the read operation. + /// + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + /// + public static ValueTask DeserializeFileAsync(string path, CancellationToken cancellationToken = default) + { + using var fs = File.OpenRead(path); + return JsonSerializer.DeserializeAsync(fs, EpicManifestParserJsonContext.Default.ManifestInfo, cancellationToken); + } + + /// Predicate to select the a single element in + /// Predicate to select the a single manifest in + /// + /// The that can be used to cancel the read operation. + /// + /// Builder for options for parsing and/or caching the manifest + /// + public Task<(FBuildPatchAppManifest Manifest, ManifestInfoElement InfoElement)> DownloadAndParseAsync( + Predicate? elementPredicate = null, Predicate? elementManifestPredicate = null, + CancellationToken cancellationToken = default, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return DownloadAndParseAsync(options, elementPredicate, elementManifestPredicate, cancellationToken); + } + + /// + /// Downloads and parses the manifest. + /// + /// Options for parsing and/or caching the manifest + /// Predicate to select the a single element in + /// Predicate to select the a single manifest in + /// + /// The that can be used to cancel the read operation. + /// + /// + /// The parsed manifest and the selected info element in a + /// + /// When a predicate fails. + /// When the manifest data fails to download. + public async Task<(FBuildPatchAppManifest Manifest, ManifestInfoElement InfoElement)> DownloadAndParseAsync( + ManifestParseOptions options, Predicate? elementPredicate = null, + Predicate? elementManifestPredicate = null, CancellationToken cancellationToken = default) + { + ManifestInfoElement element; + if (elementPredicate is null) + element = Elements[0]; + else + element = Elements.Find(elementPredicate) ?? throw new InvalidOperationException("Could not find ManifestInfoElement based on predicate"); + + ManifestInfoElementManifest elementManifest; + if (elementManifestPredicate is null) + elementManifest = element.Manifests[0]; + else + elementManifest = element.Manifests.Find(elementManifestPredicate) ?? throw new InvalidOperationException("Could not find ManifestInfoElement based on predicate"); + + string? cachePath = null; + + if (options.ManifestCacheDirectory is not null) + { + cachePath = Path.Join(options.ManifestCacheDirectory.AsSpan(), GetFileName(elementManifest.Uri)); + if (File.Exists(cachePath)) + { + var manifestBuffer = await File.ReadAllBytesAsync(cachePath, cancellationToken).ConfigureAwait(false); + var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + return (manifest, element); + } + + static ReadOnlySpan GetFileName(Uri uri) + { + var span = uri.OriginalString.AsSpan(); + return span[(span.LastIndexOf('/') + 1)..]; + } + } + + { + Uri manifestUri; + + if (elementManifest.QueryParams is { Count: not 0 }) + { + var url = new Url(elementManifest.Uri); + foreach (var queryParam in elementManifest.QueryParams) + { + url.AppendQueryParam(queryParam.Name, queryParam.Value, true, NullValueHandling.NameOnly); + } + manifestUri = url.ToUri(); + } + else + { + manifestUri = elementManifest.Uri; + } + + options.CreateDefaultClient(); + byte[] manifestBuffer; + + try + { + manifestBuffer = await options.Client!.GetByteArrayAsync(manifestUri, cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException httpEx) + { + httpEx.Data.Add("ManifestUri", manifestUri); + httpEx.Data.Add("ElementManifest", elementManifest); + httpEx.Data.Add("Element", element); + throw; + } + + var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + + if (cachePath is not null) + { + await File.WriteAllBytesAsync(cachePath, manifestBuffer, cancellationToken).ConfigureAwait(false); + } + + return (manifest, element); + } + } +} + +/// +public class ManifestInfoElement +{ + /// + public required string AppName { get; set; } + /// + public required string LabelName { get; set; } + /// + public required string BuildVersion { get; set; } + /// + public FSHAHash Hash { get; set; } + /// + public bool UseSignedUrl { get; set; } + /// + public Dictionary? Metadata { get; set; } + /// + public required List Manifests { get; set; } + + /// + public bool TryParseVersionAndCL([NotNullWhen(true)] out Version? version, out int cl) => + ManifestExtensions.TryParseVersionAndCL(BuildVersion, out version, out cl); +} + +/// +public class ManifestInfoElementManifest +{ + /// + public required Uri Uri { get; set; } + /// + public List? QueryParams { get; set; } +} + +/// +public class ManifestInfoElementManifestQueryParams +{ + /// + public required string Name { get; set; } + /// + public required string Value { get; set; } +} + + +/// +/// Source generated JSON parsers for +/// +[JsonSourceGenerationOptions(JsonSerializerDefaults.Web, Converters = [typeof(FSHAHashConverter)])] +[JsonSerializable(typeof(ManifestInfo))] +public partial class EpicManifestParserJsonContext : JsonSerializerContext; + + +/// +/// Extension methods for manifest related things +/// +public static partial class ManifestExtensions +{ + [GeneratedRegex(@"(\d+(?:\.\d+)+)-CL-(\d+)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] + internal static partial Regex VersionAndClRegex(); + + /// + /// Attempts to parse and from the . + /// + /// + /// + /// + /// if was successfully parsed; otherwise, . + public static bool TryParseVersionAndCL(string buildVersion, [NotNullWhen(true)] out Version? version, out int cl) + { + version = null; + cl = -1; + if (string.IsNullOrEmpty(buildVersion)) + return false; + + var match = VersionAndClRegex().Match(buildVersion); + if (!match.Success) + return false; + + version = Version.Parse(match.Groups[1].ValueSpan); + cl = int.Parse(match.Groups[2].ValueSpan); + return true; + } + + /// + /// Reads the HTTP content and returns the value that results from deserializing the content as JSON in an asynchronous operation. + /// + /// The content to read from. + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// The task object representing the asynchronous operation. + public static Task ReadManifestInfoAsync(this HttpContent content, CancellationToken cancellationToken = default) + => content.ReadFromJsonAsync(EpicManifestParserJsonContext.Default.ManifestInfo, cancellationToken); +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Decompressor/ManifestZlibDotNetDecompressor.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Decompressor/ManifestZlibDotNetDecompressor.cs new file mode 100644 index 0000000..33ddbab --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Decompressor/ManifestZlibDotNetDecompressor.cs @@ -0,0 +1,73 @@ +using System.IO.Compression; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Decompressor; + +/// +/// A decompressor using . +/// +public static class ManifestZlibDotNetDecompressor +{ + ///// + ///// Decompresses data buffer into destination buffer. + ///// + ///// if the decompression was successful; otherwise, . + //public static bool Decompress(object? state, byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset, int destinationLength) + //{ + // var zlibng = (Zlibng)state!; + + // var result = zlibng.Uncompress(destination.AsSpan(destinationOffset, destinationLength), + // source.AsSpan(sourceOffset, sourceLength), out int bytesWritten); + + // return result == ZlibngCompressionResult.Ok && bytesWritten == destinationLength; + //} + + /// + /// Decompresses into . + /// + /// + /// • The parameter is kept only to preserve the + /// original delegate/signature; it is not used here.
+ /// • Works on .NET 6+ with ZLibStream. + /// For earlier versions the code falls back to DeflateStream. + /// + /// + /// true when exactly bytes were + /// written; otherwise false. + /// + public static bool Decompress( + object? state, + byte[] source, + int sourceOffset, + int sourceLength, + byte[] destination, + int destinationOffset, + int destinationLength) + { + // Wrap the compressed segment in a read-only MemoryStream. + using var input = new MemoryStream(source, sourceOffset, sourceLength, writable: false); + +#if NET6_0_OR_GREATER + // ZLibStream understands the standard zlib header/footer. + using var decompressor = new ZLibStream(input, CompressionMode.Decompress, leaveOpen: false); +#else + // Older runtimes: fall back to raw deflate (no zlib header). + using var decompressor = new DeflateStream(input, CompressionMode.Decompress, leaveOpen: false); +#endif + + int totalRead = 0; + while (totalRead < destinationLength) + { + int read = decompressor.Read( + destination, + destinationOffset + totalRead, + destinationLength - totalRead); + + if (read == 0) + break; // Stream ended prematurely. + + totalRead += read; + } + + return totalRead == destinationLength; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/DeveLanCacheUI_Backend.EpicManifestParser.csproj b/DeveLanCacheUI_Backend.EpicManifestParser/DeveLanCacheUI_Backend.EpicManifestParser.csproj new file mode 100644 index 0000000..2d11934 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/DeveLanCacheUI_Backend.EpicManifestParser.csproj @@ -0,0 +1,16 @@ + + + + net9.0 + enable + enable + + + + + + + + + + diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs new file mode 100644 index 0000000..cc6c9f6 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs @@ -0,0 +1,23 @@ +using DeveLanCacheUI_Backend.EpicManifestParser.Decompressor; +using DeveLanCacheUI_Backend.EpicManifestParser.UE; + +namespace DeveLanCacheUI_Backend.EpicManifestParser +{ + public static class EpicManifestParser + { + public static FBuildPatchAppManifest Deserialize(ManifestRoData manifestBuffer) + { + var options = new ManifestParseOptions + { + //ChunkBaseUrl = "http://download.epicgames.com/Builds/UnrealEngineLauncher/CloudDir/", + //ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "chunks_v2")).FullName, + //ManifestCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "manifests_v2")).FullName, + }; + + options.Decompressor = ManifestZlibDotNetDecompressor.Decompress; + + var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + return manifest; + } + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/GlobalUsings.cs b/DeveLanCacheUI_Backend.EpicManifestParser/GlobalUsings.cs new file mode 100644 index 0000000..6d9fbc5 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/GlobalUsings.cs @@ -0,0 +1,11 @@ +#if NET9_0_OR_GREATER +global using LockObject = System.Threading.Lock; +global using ManifestData = System.Span; +global using ManifestReader = GenericReader.GenericSpanReader; +global using ManifestRoData = System.ReadOnlySpan; +#else +global using ManifestData = System.Memory; +global using ManifestRoData = System.ReadOnlyMemory; +global using ManifestReader = GenericReader.GenericBufferReader; +global using LockObject = System.Object; +#endif diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Json/BlobString.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Json/BlobString.cs new file mode 100644 index 0000000..9638196 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Json/BlobString.cs @@ -0,0 +1,59 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Json; + +[DebuggerDisplay("Value,nq")] +internal readonly struct BlobString where T : struct +{ + public T Value { get; } + public BlobString(T value) => Value = value; + + public static BlobString? Parse(ReadOnlySpan source) + { + if (source.Length == 0) return null; + T result = default; + var dest = MemoryMarshal.CreateSpan(ref Unsafe.As(ref result), Unsafe.SizeOf()); + + // Make sure the buffer is at least half the size and that the string is an + // even number of characters long + if (dest.Length >= (uint32)(source.Length / 3) && source.Length % 3 == 0) + { + Span convBuffer = stackalloc uint8[4]; + convBuffer[3] = 0; + + int32 WriteIndex = 0; + // Walk the string 3 chars at a time + for (int32 Index = 0; Index < source.Length; Index += 3, WriteIndex++) + { + convBuffer[0] = source[Index]; + convBuffer[1] = source[Index + 1]; + convBuffer[2] = source[Index + 2]; + dest[WriteIndex] = uint8.Parse(convBuffer); + } + return result; + } + return null; + } + + public static implicit operator BlobString(T value) + { + return new BlobString(value); + } + + public static explicit operator T?(BlobString? holder) + { + return holder?.Value; + } +} + +internal sealed class BlobStringConverter : JsonConverter?> where T : struct +{ + public override BlobString? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => BlobString.Parse(reader.ValueSpan); + public override void Write(Utf8JsonWriter writer, BlobString? value, JsonSerializerOptions options) + => throw new NotSupportedException(); +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Json/JsonNodeExtensions.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Json/JsonNodeExtensions.cs new file mode 100644 index 0000000..7d9dd91 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Json/JsonNodeExtensions.cs @@ -0,0 +1,54 @@ +using DeveLanCacheUI_Backend.EpicManifestParser.UE; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Json; + +internal static class JsonNodeExtensions +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + Converters = + { + new FGuidConverter(), + new FSHAHashConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + } + }; + + public static T GetBlob(this JsonNode? node, T defaultValue = default) where T : struct + { + return node.Deserialize?>(SerializerOptions)?.Value ?? defaultValue; + } + + public static FGuid GetFGuid(this JsonNode? node) + { + return node.Deserialize(SerializerOptions); + } + + public static FSHAHash GetSha(this JsonNode? node) + { + return node.Deserialize(SerializerOptions); + } + + public static string GetString(this JsonNode? node, string defaultValue = "") + { + return node?.GetValue() ?? defaultValue; + } + + public static T Get(this JsonNode? node, T defaultValue = default!) + { + return node is null ? defaultValue : node.GetValue(); + } + + public static T Parse(this JsonNode? node, T defaultValue = default!) + { + return node is null ? defaultValue : node.Deserialize() ?? defaultValue; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/ManifestParseOptions.cs b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestParseOptions.cs new file mode 100644 index 0000000..b491754 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestParseOptions.cs @@ -0,0 +1,87 @@ +using System.Net; + +namespace DeveLanCacheUI_Backend.EpicManifestParser; +// ReSharper disable UseSymbolAlias + +/// +/// Options/Configuration for parsing manifests +/// +public class ManifestParseOptions +{ + /// + /// Zlib decompress delegate. + /// + public delegate bool DecompressDelegate(object? state, byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset, int destinationLength); + + /// + /// Zlib decompress delegate, defaults to . + /// + public DecompressDelegate? Decompressor { get; set; } = ManifestZlibStreamDecompressor.Decompress; + + /// + /// Optional state that gets passed to the delegate. + /// + public object? DecompressorState { get; set; } + + /// + /// Required for downloading, must have a leading slash! + /// + /// + /// Example: http://epicgames-download1.akamaized.net/Builds/Fortnite/CloudDir/
+ /// Distributionpoints can be found here: here. + /// + public string? ChunkBaseUrl { get; set; } + + /// + /// Your own (optional) used for downloading, must not have a ! + /// + public HttpClient? Client { get; set; } + + /// + /// Buffer size for downloading chunks, defaults to 2097152 bytes (2 MiB). + /// + public int ChunkDownloadBufferSize { get; set; } = 2097152; + + /// + /// Optional for caching chunks, very recommended. + /// + public string? ChunkCacheDirectory { get; set; } + + /// + /// Whether or not to cache the chunks 1:1 as they were downloaded, defaults to . + /// + public bool CacheChunksAsIs { get; set; } + + /// + /// Optional for caching manifests when using . + /// + public string? ManifestCacheDirectory { get; set; } + + /// + /// Creates a default and also sets to its instance. + /// + /// The created . + public HttpClient CreateDefaultClient() + { + if (Client is not null) + return Client; + + var handler = new SocketsHttpHandler + { + UseCookies = false, + UseProxy = false, + AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip, + MaxConnectionsPerServer = 256 + }; + Client = new HttpClient(handler) + { + DefaultRequestVersion = new Version(1, 1), + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + Timeout = TimeSpan.FromSeconds(30) + }; + Client.DefaultRequestHeaders.Accept.ParseAdd("*/*"); + Client.DefaultRequestHeaders.UserAgent.ParseAdd("EpicGamesLauncher/16.13.0-36938137+++Portal+Release-Live Windows/10.0.26100.1.256.64bit"); + Client.DefaultRequestHeaders.ConnectionClose = false; + return Client; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/ManifestZlibStreamDecompressor.cs b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestZlibStreamDecompressor.cs new file mode 100644 index 0000000..710f6c1 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestZlibStreamDecompressor.cs @@ -0,0 +1,22 @@ +using System.IO.Compression; + +namespace DeveLanCacheUI_Backend.EpicManifestParser; + +/// +/// The default decompressor using . +/// +public static class ManifestZlibStreamDecompressor +{ + /// + /// Decompresses data buffer into destination buffer. + /// + /// if the decompression was successful; otherwise, . + public static bool Decompress(object? state, byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset, int destinationLength) + { + using var destinationMs = new MemoryStream(destination, destinationOffset, destinationLength, true, true); + using var sourceMs = new MemoryStream(source, sourceOffset, sourceLength, false, true); + using var zlibStream = new ZLibStream(sourceMs, CompressionMode.Decompress); + zlibStream.CopyTo(destinationMs); + return destinationMs.Position == destinationLength; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkDataListVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkDataListVersion.cs new file mode 100644 index 0000000..ea3e9f5 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkDataListVersion.cs @@ -0,0 +1,10 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EChunkDataListVersion : uint8 +{ + Original = 0, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkHashFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkHashFlags.cs new file mode 100644 index 0000000..efa5522 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkHashFlags.cs @@ -0,0 +1,13 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +[Flags] +internal enum EChunkHashFlags : uint8 +{ + None = 0x00, + + // Flag for FRollingHash class used, stored in RollingHash on header. + RollingPoly64 = 0x01, + + // Flag for FSHA1 class used, stored in SHAHash on header. + Sha1 = 0x02, +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkStorageFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkStorageFlags.cs new file mode 100644 index 0000000..7e610da --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkStorageFlags.cs @@ -0,0 +1,13 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +[Flags] +internal enum EChunkStorageFlags : uint8 +{ + None = 0x00, + + // Flag for compressed data. + Compressed = 0x01, + + // Flag for encrypted. If also compressed, decrypt first. Encryption will ruin compressibility. + Encrypted = 0x02, +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkVersion.cs new file mode 100644 index 0000000..10dba05 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkVersion.cs @@ -0,0 +1,13 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EChunkVersion : uint32 +{ + Invalid = 0, + Original, + StoresShaAndHashType, + StoresDataSizeUncompressed, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFeatureLevel.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFeatureLevel.cs new file mode 100644 index 0000000..6c44335 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFeatureLevel.cs @@ -0,0 +1,133 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE EFeatureLevel enum +/// +public enum EFeatureLevel +{ + /// + /// The original version. + /// + Original = 0, + /// + /// Support for custom fields. + /// + CustomFields, + /// + /// Started storing the version number. + /// + StartStoringVersion, + /// + /// Made after data files where renamed to include the hash value, these chunks now go to ChunksV2. + /// + DataFileRenames, + /// + /// Manifest stores whether build was constructed with chunk or file data. + /// + StoresIfChunkOrFileData, + /// + /// Manifest stores group number for each chunk/file data for reference so that external readers don't need to know how to calculate them. + /// + StoresDataGroupNumbers, + /// + /// Added support for chunk compression, these chunks now go to ChunksV3. NB: Not File Data Compression yet. + /// + ChunkCompressionSupport, + /// + /// Manifest stores product prerequisites info. + /// + StoresPrerequisitesInfo, + /// + /// Manifest stores chunk download sizes. + /// + StoresChunkFileSizes, + /// + /// Manifest can optionally be stored using UObject serialization and compressed. + /// + StoredAsCompressedUClass, + /// + /// Removed and never used. + /// + UNUSED_0, + /// + /// Removed and never used. + /// + UNUSED_1, + /// + /// Manifest stores chunk data SHA1 hash to use in place of data compare, for faster generation. + /// + StoresChunkDataShaHashes, + /// + /// Manifest stores Prerequisite Ids. + /// + StoresPrerequisiteIds, + /// + /// The first minimal binary format was added. UObject classes will no longer be saved out when binary selected. + /// + StoredAsBinaryData, + /// + /// Temporary level where manifest can reference chunks with dynamic window size, but did not serialize them. Chunks from here onwards are stored in ChunksV4. + /// + VariableSizeChunksWithoutWindowSizeChunkInfo, + /// + /// Manifest can reference chunks with dynamic window size, and also serializes them. + /// + VariableSizeChunks, + /// + /// Manifest uses a build id generated from its metadata. + /// + UsesRuntimeGeneratedBuildId, + /// + /// Manifest uses a build id generated unique at build time, and stored in manifest. + /// + UsesBuildTimeGeneratedBuildId, + + /// + /// Undocumented in UE + /// + Unknown1, + /// + /// Undocumented in UE + /// + Unknown2, + /// + /// Used for fortnite currently + /// + Unknown3, + + /// + /// !! Always after the latest version entry, signifies the latest version plus 1 to allow the following Latest alias. + /// + LatestPlusOne = UsesBuildTimeGeneratedBuildId + 1, + /// + /// An alias for the actual latest version value. + /// + Latest = LatestPlusOne - 1, + /// + /// An alias to provide the latest version of a manifest supported by file data (nochunks). + /// + LatestNoChunks = StoresChunkFileSizes, + /// + /// An alias to provide the latest version of a manifest supported by a json serialized format. + /// + LatestJson = StoresPrerequisiteIds, + /// + /// An alias to provide the first available version of optimised delta manifest saving. + /// + FirstOptimisedDelta = UsesRuntimeGeneratedBuildId, + + /// + /// More aliases, but this time for values that have been renamed + /// + StoresUniqueBuildId = UsesRuntimeGeneratedBuildId, + + /// + /// JSON manifests were stored with a version of 255 during a certain CL range due to a bug. + /// We will treat this as being StoresChunkFileSizes in code. + /// + BrokenJsonVersion = 255, + /// + /// This is for UObject default, so that we always serialize it. + /// + Invalid = -1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileManifestListVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileManifestListVersion.cs new file mode 100644 index 0000000..71062e1 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileManifestListVersion.cs @@ -0,0 +1,10 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EFileManifestListVersion : uint8 +{ + Original = 0, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileMetaFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileMetaFlags.cs new file mode 100644 index 0000000..d0971ae --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileMetaFlags.cs @@ -0,0 +1,25 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE EFileMetaFlags enum +/// +[Flags] +public enum EFileMetaFlags : uint8 +{ + /// + /// None + /// + None = 0, + /// + /// Flag for readonly file. + /// + ReadOnly = 1, + /// + /// Flag for natively compressed. + /// + Compressed = 1 << 1, + /// + /// Flag for unix executable. + /// + UnixExecutable = 1 << 2 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestMetaVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestMetaVersion.cs new file mode 100644 index 0000000..2cacf4c --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestMetaVersion.cs @@ -0,0 +1,11 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EManifestMetaVersion : uint8 +{ + Original = 0, + SerialisesBuildId, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestStorageFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestStorageFlags.cs new file mode 100644 index 0000000..de35dd6 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestStorageFlags.cs @@ -0,0 +1,12 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +[Flags] +internal enum EManifestStorageFlags : uint8 +{ + // Stored as raw data. + None = 0, + // Flag for compressed data. + Compressed = 1, + // Flag for encrypted. If also compressed, decrypt first. Encryption will ruin compressibility. + Encrypted = 1 << 1, +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FBuildPatchAppManifest.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FBuildPatchAppManifest.cs new file mode 100644 index 0000000..975e9ae --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FBuildPatchAppManifest.cs @@ -0,0 +1,508 @@ +using AsyncKeyedLock; +using DeveLanCacheUI_Backend.EpicManifestParser.Json; +using System.Buffers; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FBuildPatchAppManifest struct +/// +public class FBuildPatchAppManifest +{ + /// + public FManifestMeta Meta { get; internal set; } = null!; + /// + public IReadOnlyList ChunkList { get; internal set; } = null!; + /// + public IReadOnlyList Files { get; internal set; } = null!; + /// + public IReadOnlyList CustomFields { get; internal set; } = null!; + /// + public IReadOnlyDictionary Chunks { get; internal set; } = null!; + + /// + public int64 TotalBuildSize { get; internal set; } + /// + public int64 TotalDownloadSize { get; internal set; } + + internal ManifestParseOptions Options { get; init; } = null!; + internal AsyncKeyedLocker ChunksLocker { get; set; } = null!; + + internal FBuildPatchAppManifest() { } + + /// + /// Finds a file by . + /// + /// The filename to find. + /// The type to compare the filename. + /// The instance if the the file was found; otherwise, . + public FFileManifest? FindFile(string fileName, StringComparison comparisonType = StringComparison.Ordinal) + => TryFindFile(fileName, comparisonType, out var file) ? file : null; + + /// + /// Tries to find a file by using to compare it. + /// + /// The filename to find. + /// The find result. + /// if the the file was found; otherwise, . + public bool TryFindFile(string fileName, [NotNullWhen(true)] out FFileManifest? fileManifest) + => TryFindFile(fileName, StringComparison.Ordinal, out fileManifest); + + /// + /// Tries to find a file by . + /// + /// The filename to find. + /// The type to compare the filename. + /// The find result. + /// if the the file was found; otherwise, . + public bool TryFindFile(string fileName, StringComparison comparisonType, [NotNullWhen(true)] out FFileManifest? fileManifest) + { + foreach (var file in Files) + { + if (!file.FileName.Equals(fileName, comparisonType)) + continue; + + fileManifest = file; + return true; + } + + fileManifest = null; + return false; + } + + /// + /// Get the chunk sub-directory name + /// + public string GetChunkSubdir() => Meta.FeatureLevel switch + { + > EFeatureLevel.StoredAsBinaryData => "ChunksV4", + > EFeatureLevel.StoresDataGroupNumbers => "ChunksV3", + > EFeatureLevel.StartStoringVersion => "ChunksV2", + _ => "Chunks" + }; + + /// + /// Helper function to decide whether the passed in data is a JSON string we expect to deserialize a manifest from + /// + /// if the is JSON; otherwise, . + public static bool IsJson(ManifestRoData dataInput) + { + // The best we can do is look for the mandatory first character open curly brace, + // it will be within the first 4 characters (may have BOM) + var span = dataInput +#if !NET9_0_OR_GREATER + .Span +#endif + ; + for (var idx = 0; idx < 4 && idx < span.Length; ++idx) + { + if (span[idx] == '{') + { + return true; + } + } + return false; + } + + /// + /// Deserializes a binary or JSON manifest + /// + /// + public static FBuildPatchAppManifest Deserialize(ManifestRoData dataInput, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return Deserialize(dataInput, options); + } + + /// + /// Deserializes a JSON manifest + /// + /// The span to parse from + /// Builder for options/configuration to parse + public static FBuildPatchAppManifest DeserializeJson(ManifestRoData dataInput, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return DeserializeJson(dataInput, options); + } + + /// + /// Deserializes a binary manifest + /// + /// The span to parse from + /// Builder for options/configuration to parse + /// Manifest is encrypted or older than + /// Data is compressed and zlib-ng instance was null + /// Error while parsing + /// Hashes do not match + public static FBuildPatchAppManifest DeserializeBinary(ManifestRoData dataInput, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return DeserializeBinary(dataInput, options); + } + + /// + /// Deserializes a binary or JSON manifest + /// + /// + public static FBuildPatchAppManifest Deserialize(ManifestRoData dataInput, ManifestParseOptions options) + { + return IsJson(dataInput) ? DeserializeJson(dataInput, options) : DeserializeBinary(dataInput, options); + } + + /// + /// Deserializes a JSON manifest + /// + /// The span to parse from + /// Options/Configuration to parse + public static FBuildPatchAppManifest DeserializeJson(ManifestRoData dataInput, ManifestParseOptions options) + { + var reader = JsonNode.Parse(dataInput +#if !NET9_0_OR_GREATER + .Span +#endif + )!.AsObject(); + + var featureLevel = reader["ManifestFileVersion"].GetBlob(EFeatureLevel.CustomFields); + if (featureLevel == EFeatureLevel.BrokenJsonVersion) + featureLevel = EFeatureLevel.StoresChunkFileSizes; + + var meta = new FManifestMeta + { + FeatureLevel = featureLevel, + AppID = reader["AppID"].GetBlob(), + AppName = reader["AppNameString"].GetString(), + BuildVersion = reader["BuildVersionString"].GetString(), + LaunchExe = reader["LaunchExeString"].GetString(), + LaunchCommand = reader["LaunchCommand"].GetString(), + PrereqName = reader["PrereqName"].GetString(), + PrereqPath = reader["PrereqPath"].GetString(), + PrereqArgs = reader["PrereqArgs"].GetString(), + UninstallExe = "", + UninstallCommand = "", + }; + + var jsonFileManifestList = reader["FileManifestList"]!.AsArray(); + var fileManifests = new FFileManifest[jsonFileManifestList.Count]; + var fileManifestsSpan = fileManifests.AsSpan(); + + //var allDataGuids = new HashSet(); + var mutableChunkInfoLookup = new Dictionary(); + + for (var i = 0; i < fileManifestsSpan.Length; i++) + { + var jsonFileManifest = jsonFileManifestList[i]!; + var fileManifest = fileManifestsSpan[i] = new FFileManifest + { + FileName = jsonFileManifest["Filename"].GetString(), + FileHash = jsonFileManifest["FileHash"].GetBlob(), + InstallTags = jsonFileManifest["InstallTags"].Parse([]), + SymlinkTarget = jsonFileManifest["SymlinkTarget"].GetString() + }; + var jsonFileChunkParts = jsonFileManifest["FileChunkParts"]!.AsArray(); + fileManifest.ChunkPartsArray = new FChunkPart[jsonFileChunkParts.Count]; + var chunkPartsSpan = fileManifest.ChunkPartsArray.AsSpan(); + for (var j = 0; j < chunkPartsSpan.Length; j++) + { + var jsonFileChunkPart = jsonFileChunkParts[j]!; + var chunkPartGuid = jsonFileChunkPart["Guid"].GetFGuid(); + var chunkPartOffset = jsonFileChunkPart["Offset"].GetBlob(); + var chunkPartSize = jsonFileChunkPart["Size"].GetBlob(); + chunkPartsSpan[j] = new FChunkPart(chunkPartGuid, chunkPartOffset, chunkPartSize); + + ref var lookupChunk = ref CollectionsMarshal.GetValueRefOrAddDefault(mutableChunkInfoLookup, chunkPartGuid, out var exists); + if (!exists) + { + lookupChunk = new FChunkInfo + { + Guid = chunkPartGuid + }; + } + } + + if (jsonFileManifest["bIsUnixExecutable"].Get()) + fileManifest.FileMetaFlags |= EFileMetaFlags.UnixExecutable; + if (jsonFileManifest["bIsReadOnly"].Get()) + fileManifest.FileMetaFlags |= EFileMetaFlags.ReadOnly; + if (jsonFileManifest["bIsCompressed"].Get()) + fileManifest.FileMetaFlags |= EFileMetaFlags.Compressed; + } + + var chunkList = new FChunkInfo[mutableChunkInfoLookup.Count]; + var chunkListSpan = chunkList.AsSpan(); + var chunkIndex = 0; + foreach (var chunk in mutableChunkInfoLookup.Values) + { + chunkListSpan[chunkIndex++] = chunk; + } + + var hasChunkHashList = false; + var jsonChunkHashListNode = reader["ChunkHashList"]; + if (jsonChunkHashListNode is not null) + { + var jsonChunkHashList = jsonChunkHashListNode.AsObject(); + + foreach (var (guidString, jsonChunkHash) in jsonChunkHashList) + { + var guid = new FGuid(guidString); + var chunkHash = jsonChunkHash.GetBlob(); + mutableChunkInfoLookup[guid].Hash = chunkHash; + } + + hasChunkHashList = true; + } + + var jsonChunkShaListNode = reader["ChunkShaList"]; + if (jsonChunkShaListNode is not null) + { + var jsonChunkShaList = jsonChunkShaListNode.AsObject(); + + foreach (var (guidString, jsonSha) in jsonChunkShaList) + { + var guid = new FGuid(guidString); + var chunkSha = jsonSha.GetSha(); + mutableChunkInfoLookup[guid].ShaHash = chunkSha; + } + } + + var prereqIds = reader["PrereqIds"].Deserialize(); + if (prereqIds is null) + { + // TODO: https://github.com/EpicGames/UnrealEngine/blob/8c31706601135aadf2f957fb76e2af46f04a8ef9/Engine/Source/Runtime/Online/BuildPatchServices/Private/BuildPatchManifest.cpp#L602 + meta.PrereqIds = []; + } + else + { + meta.PrereqIds = prereqIds; + } + + var jsonDataGroupListNode = reader["DataGroupList"]; + if (jsonDataGroupListNode is not null) + { + var jsonDataGroupList = jsonDataGroupListNode.AsObject(); + + foreach (var (guidString, jsonDataGroup) in jsonDataGroupList) + { + var guid = new FGuid(guidString); + var dataGroup = jsonDataGroup.GetBlob(); + mutableChunkInfoLookup[guid].GroupNumber = dataGroup; + } + } + else + { + // TODO: https://github.com/EpicGames/UnrealEngine/blob/8c31706601135aadf2f957fb76e2af46f04a8ef9/Engine/Source/Runtime/Online/BuildPatchServices/Private/BuildPatchManifest.cpp#L635 + // https://github.com/EpicGames/UnrealEngine/blob/8c31706601135aadf2f957fb76e2af46f04a8ef9/Engine/Source/Runtime/Core/Private/Misc/Crc.cpp#L592 + } + + var hasChunkFilesizeList = false; + var jsonChunkFilesizeListNode = reader["ChunkFilesizeList"]; + if (jsonChunkFilesizeListNode is not null) + { + var jsonChunkFilesizeList = jsonChunkFilesizeListNode.AsObject(); + + foreach (var (guidString, jsonFileSize) in jsonChunkFilesizeList) + { + var guid = new FGuid(guidString); + var fileSize = jsonFileSize.GetBlob(); + mutableChunkInfoLookup[guid].FileSize = fileSize; + } + + hasChunkFilesizeList = true; + } + + if (!hasChunkFilesizeList) + { + // Missing chunk list, version before we saved them compressed. Assume original fixed chunk size of 1 MiB. + foreach (var chunk in chunkListSpan) + { + chunk.FileSize = 1048576; + } + } + + if (reader.TryGetPropertyValue("bIsFileData", out var jsonIsFileData)) + { + meta.bIsFileData = jsonIsFileData.Get(); + } + else + { + meta.bIsFileData = !hasChunkHashList; + } + + FCustomField[]? customFields = null; + var jsonCustomFieldsNode = reader["CustomFields"]; + if (jsonCustomFieldsNode is not null) + { + var jsonCustomFields = jsonCustomFieldsNode.AsObject(); + customFields = new FCustomField[jsonCustomFields.Count]; + var customFieldIndex = 0; + + foreach (var (name, jsonValue) in jsonCustomFields) + { + customFields[customFieldIndex++] = new FCustomField + { + Name = name, + Value = jsonValue.GetString() + }; + } + } + + meta.BuildId = FManifestMeta.GetBackwardsCompatibleBuildId(meta); + + var manifest = new FBuildPatchAppManifest + { + Meta = meta, + ChunkList = chunkList, + Files = fileManifests, + CustomFields = customFields ?? [], + Chunks = mutableChunkInfoLookup, + Options = options + }; + manifest.PostSetup(); + + // FileDataList.OnPostLoad(); + { + Array.Sort(fileManifests); + for (var i = 0; i < fileManifestsSpan.Length; i++) + { + var file = fileManifestsSpan[i]; + file.Manifest = manifest; + foreach (var chunkPart in file.ChunkPartsArray.AsSpan()) + { + file.FileSize += chunkPart.Size; + } + } + } + + return manifest; + } + + /// + /// Deserializes a binary manifest + /// + /// The span to parse from + /// Options/Configuration to parse + /// Manifest is encrypted or older than + /// Data is compressed and zlib-ng instance was null + /// Error while parsing + /// Hashes do not match + public static FBuildPatchAppManifest DeserializeBinary(ManifestRoData dataInput, ManifestParseOptions options) + { + var fileReader = new ManifestReader(dataInput); + byte[]? manifestRawDataBuffer = null; + + try + { + var header = new FManifestHeader(ref fileReader); + + if (header.Version < EFeatureLevel.StoredAsBinaryData) + throw new NotSupportedException("Manifests below feature level StoredAsBinaryData are not supported"); + if (header.StoredAs.HasFlag(EManifestStorageFlags.Encrypted)) + throw new NotSupportedException("Encrypted manifests are not supported"); + if (header.StoredAs.HasFlag(EManifestStorageFlags.Compressed) && options.Decompressor is null) + throw new InvalidOperationException("Data is compressed and decompressor delegate was null"); + + ManifestData manifestRawData; + + if (header.StoredAs.HasFlag(EManifestStorageFlags.Compressed)) + { + manifestRawDataBuffer = ArrayPool.Shared.Rent(header.DataSizeCompressed + header.DataSizeUncompressed); + manifestRawData = manifestRawDataBuffer +#if NET9_0_OR_GREATER + .AsSpan +#else + .AsMemory +#endif + (header.DataSizeCompressed, header.DataSizeUncompressed); + + var manifestCompressedData = manifestRawDataBuffer.AsSpan(0, header.DataSizeCompressed); + fileReader.Read(manifestCompressedData); + + var result = options.Decompressor!.Invoke( + options.DecompressorState, + manifestRawDataBuffer, 0, header.DataSizeCompressed, + manifestRawDataBuffer, header.DataSizeCompressed, header.DataSizeUncompressed); + if (!result) + throw new FileLoadException("Failed to uncompress data"); + } + else if (header.StoredAs == EManifestStorageFlags.None) + { + manifestRawDataBuffer = ArrayPool.Shared.Rent(header.DataSizeCompressed); + manifestRawData = manifestRawDataBuffer +#if NET9_0_OR_GREATER + .AsSpan +#else + .AsMemory +#endif + (0, header.DataSizeCompressed); + fileReader.Read(manifestRawData +#if !NET9_0_OR_GREATER + .Span +#endif + ); + } + else + { + throw new UnreachableException("Manifest has invalid or unknown storage flags"); + } + + var hash = FSHAHash.Compute(manifestRawData +#if !NET9_0_OR_GREATER + .Span +#endif + ); + if (header.SHAHash != hash) + throw new InvalidDataException($"Hash does not match. expected: {header.SHAHash}, actual: {hash}"); + + var reader = new ManifestReader(manifestRawData); + var chunks = new Dictionary(); + var manifest = new FBuildPatchAppManifest + { + Chunks = chunks, + Options = options + }; + manifest.Meta = new FManifestMeta(ref reader); + manifest.ChunkList = FChunkInfo.ReadChunkDataList(ref reader, chunks); + manifest.Files = FFileManifest.ReadFileDataList(ref reader, manifest); + manifest.CustomFields = FCustomField.ReadCustomFields(ref reader); + manifest.PostSetup(); + return manifest; + } + finally + { + if (manifestRawDataBuffer is not null) + ArrayPool.Shared.Return(manifestRawDataBuffer); + } + } + + private void PostSetup() + { + foreach (var file in Files) + { + TotalBuildSize += file.FileSize; + } + + foreach (var chunk in ChunkList) + { + TotalDownloadSize += chunk.FileSize; + } + + if (!string.IsNullOrEmpty(Options.ChunkBaseUrl)) + { + ChunksLocker = new AsyncKeyedLocker(lockerOptions => + { + lockerOptions.MaxCount = 1; + lockerOptions.PoolSize = 128; + lockerOptions.PoolInitialFill = 64; + }); + + Options.CreateDefaultClient(); + } + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkHeader.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkHeader.cs new file mode 100644 index 0000000..4e80656 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkHeader.cs @@ -0,0 +1,113 @@ +using System.Runtime.CompilerServices; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal struct FChunkHeader +{ + public const uint32 Magic = 0xB1FE3AA2; + + /// + /// The version of this header data. + /// + public EChunkVersion Version; + /// + /// The size of this header. + /// + public int32 HeaderSize; + /*/// + /// The GUID for this data. + /// + public FGuid Guid;*/ + /// + /// The size of this data compressed. + /// + public int32 DataSizeCompressed; + /// + /// The size of this data uncompressed. + /// + public int32 DataSizeUncompressed; + /// + /// How the chunk data is stored. + /// + public EChunkStorageFlags StoredAs; + /*/// + /// What type of hash we are using. + /// + public EChunkHashFlags HashType; + /// + /// The FRollingHash hashed value for this chunk data. + /// + public uint64 RollingHash; + /// + /// The FSHA hashed value for this chunk data. + /// + public FSHAHash SHAHash;*/ + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static FChunkHeader Parse(ManifestData data) + { + var reader = new ManifestReader(data); + return new FChunkHeader(ref reader); + } + + internal FChunkHeader(ref ManifestReader reader) + { + var startPos = reader.Position; + var archiveSizeLeft = reader.Length - startPos; + var versionSizesSpan = ChunkHeaderVersionSizes.AsSpan(); + var expectedSerializedBytes = versionSizesSpan[(int32)EChunkVersion.Original]; + + if (archiveSizeLeft >= expectedSerializedBytes) + { + var magic = reader.Read(); + if (magic != Magic) + throw new FileLoadException($"invalid chunk magic: 0x{magic:X}"); + + Version = reader.Read(); + HeaderSize = reader.Read(); + DataSizeCompressed = reader.Read(); + + //Guid = reader.Read(); + //RollingHash = reader.Read(); + reader.Position += FGuid.Size + sizeof(uint64); + + StoredAs = reader.Read(); + DataSizeUncompressed = 1024 * 1024; + + if (Version >= EChunkVersion.StoresShaAndHashType) + { + expectedSerializedBytes = versionSizesSpan[(int32)EChunkVersion.StoresShaAndHashType]; + if (archiveSizeLeft >= expectedSerializedBytes) + { + //SHAHash = reader.Read(); + //HashType = reader.Read(); + reader.Position += FSHAHash.Size + sizeof(EChunkHashFlags); + } + + if (Version >= EChunkVersion.StoresDataSizeUncompressed) + { + expectedSerializedBytes = versionSizesSpan[(int32)EChunkVersion.StoresDataSizeUncompressed]; + if (archiveSizeLeft >= expectedSerializedBytes) + { + DataSizeUncompressed = reader.Read(); + } + } + } + } + + var success = reader.Position - startPos == expectedSerializedBytes; + reader.Position = startPos + HeaderSize; + } + + private static readonly uint32[] ChunkHeaderVersionSizes = + [ + // Dummy for indexing. + 0, + // Original is 41 bytes (32b Magic, 32b Version, 32b HeaderSize, 32b DataSizeCompressed, 4x32b GUID, 64b Hash, 8b StoredAs). + 41, + // StoresShaAndHashType is 62 bytes (328b Original, 160b SHA1, 8b HashType). + 62, + // StoresDataSizeUncompressed is 66 bytes (496b StoresShaAndHashType, 32b DataSizeUncompressed). + 66 + ]; +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkInfo.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkInfo.cs new file mode 100644 index 0000000..28e0ff4 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkInfo.cs @@ -0,0 +1,331 @@ +using System.Buffers; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FChunkInfo struct +/// +public sealed class FChunkInfo +{ + /// + /// The GUID for this data. + /// + public FGuid Guid { get; internal set; } + /// + /// The FRollingHash hashed value for this chunk data. + /// + public uint64 Hash { get; internal set; } + /// + /// The FSHA hashed value for this chunk data. + /// + public FSHAHash ShaHash { get; internal set; } + /// + /// The group number this chunk divides into. + /// + public uint8 GroupNumber { get; internal set; } + /// + /// The window size for this chunk. + /// + public uint32 WindowSize { get; internal set; } + /// + /// The file download size for this chunk. + /// + public int64 FileSize { get; internal set; } + + /// + /// + /// Url to download this chunk + /// + public string GetUrl(FBuildPatchAppManifest manifest) => + $"{manifest.Options.ChunkBaseUrl}{manifest.GetChunkSubdir()}/{GroupNumber:D2}/{Hash:X16}_{Guid}.chunk"; + + /// + /// + /// to download this chunk + /// + public Uri GetUri(FBuildPatchAppManifest manifest) => new(GetUrl(manifest), UriKind.Absolute); + + internal string? CachePath { get; set; } + + internal static FChunkInfo[] ReadChunkDataList(ref ManifestReader reader, Dictionary chunksDict) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + var elementCount = reader.Read(); + + var chunks = new FChunkInfo[elementCount]; + var chunksSpan = chunks.AsSpan(); + + chunksDict.EnsureCapacity(elementCount); + + if (dataVersion >= EChunkDataListVersion.Original) + { + for (var i = 0; i < elementCount; i++) + { + var chunk = new FChunkInfo(); + chunk.Guid = reader.Read(); + chunksSpan[i] = chunk; + chunksDict.Add(chunk.Guid, chunk); + } + for (var i = 0; i < elementCount; i++) + chunksSpan[i].Hash = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].ShaHash = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].GroupNumber = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].WindowSize = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].FileSize = reader.Read(); + } + else + { + var defaultChunk = new FChunkInfo + { + WindowSize = 1048576 + }; + chunksSpan.Fill(defaultChunk); + } + + reader.Position = startPos + dataSize; + return chunks; + } + + [SuppressMessage("ReSharper", "UseSymbolAlias")] + internal async Task ReadDataAsIsAsync(byte[] destination, FBuildPatchAppManifest manifest, CancellationToken cancellationToken = default) + { + var fileSize = 0; + var shouldCache = manifest.Options.ChunkCacheDirectory is not null; + string? cachePath = null; + + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + fileSize = (int)RandomAccess.GetLength(fileHandle); + await RandomAccess.ReadAsync(fileHandle, destination.AsMemory(0, fileSize), 0, cancellationToken).ConfigureAwait(false); + } + else + { + using var _ = await manifest.ChunksLocker.LockAsync(Guid, cancellationToken).ConfigureAwait(false); + + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + fileSize = (int)RandomAccess.GetLength(fileHandle); + await RandomAccess.ReadAsync(fileHandle, destination.AsMemory(0, fileSize), 0, cancellationToken).ConfigureAwait(false); + } + else if (shouldCache) + { + cachePath = Path.Combine(manifest.Options.ChunkCacheDirectory!, $"v2_{Hash:X16}_{Guid}.chunk"); + if (File.Exists(cachePath)) + { + CachePath = cachePath; + using var fileHandle = File.OpenHandle(CachePath); + fileSize = (int)RandomAccess.GetLength(fileHandle); + await RandomAccess.ReadAsync(fileHandle, destination.AsMemory(0, fileSize), 0, cancellationToken).ConfigureAwait(false); + } + } + + if (fileSize == 0) + { + var uri = GetUri(manifest); + var destMs = new MemoryStream(destination, 0, destination.Length, true); + using var res = await manifest.Options.Client!.GetAsync(uri, cancellationToken).ConfigureAwait(false); + EnsureSuccessStatusCode(res, uri); + await res.Content.CopyToAsync(destMs, cancellationToken).ConfigureAwait(false); + fileSize = (int)destMs.Position; + + if (shouldCache) + { + using (var fileHandle = File.OpenHandle(cachePath!, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, fileSize)) + { + await RandomAccess.WriteAsync(fileHandle, new ReadOnlyMemory(destination, 0, fileSize), 0, cancellationToken).ConfigureAwait(false); + RandomAccess.FlushToDisk(fileHandle); + } + CachePath = cachePath; + } + } + } + + var header = FChunkHeader.Parse(new ManifestData(destination, 0, fileSize)); + + if (header.StoredAs == EChunkStorageFlags.None) + { + Unsafe.CopyBlockUnaligned(ref destination[0], ref destination[header.HeaderSize], (uint)header.DataSizeCompressed); + return header.DataSizeCompressed; + } + + if (header.StoredAs.HasFlag(EChunkStorageFlags.Encrypted)) + throw new NotSupportedException("Encrypted chunks are not supported"); + if (!header.StoredAs.HasFlag(EChunkStorageFlags.Compressed)) + throw new UnreachableException("Unknown/new chunk ChunkStorageFlag"); + if (manifest.Options.Decompressor is null) + throw new InvalidOperationException("Data is compressed and decompressor delegate was null"); + + // cant uncompress in-place + var poolBuffer = ArrayPool.Shared.Rent(header.DataSizeCompressed); + + try + { + Unsafe.CopyBlockUnaligned(ref poolBuffer[0], ref destination[header.HeaderSize], (uint)header.DataSizeCompressed); + + var result = manifest.Options.Decompressor.Invoke( + manifest.Options.DecompressorState, + poolBuffer, 0, header.DataSizeCompressed, + destination, 0, header.DataSizeUncompressed); + if (!result) + throw new FileLoadException("Failed to uncompress data"); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + + return header.DataSizeUncompressed; + } + + [SuppressMessage("ReSharper", "UseSymbolAlias")] + internal async Task ReadDataAsync(byte[] buffer, int offset, int count, int chunkPartOffset, FBuildPatchAppManifest manifest, CancellationToken cancellationToken = default) + { + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + return await RandomAccess.ReadAsync(fileHandle, buffer.AsMemory(offset, count), chunkPartOffset, cancellationToken).ConfigureAwait(false); + } + + using var _ = await manifest.ChunksLocker.LockAsync(Guid, cancellationToken).ConfigureAwait(false); + + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + return await RandomAccess.ReadAsync(fileHandle, buffer.AsMemory(offset, count), chunkPartOffset, cancellationToken).ConfigureAwait(false); + } + + var shouldCache = manifest.Options.ChunkCacheDirectory is not null; + string? cachePath = null; + + if (shouldCache) + { + cachePath = Path.Combine(manifest.Options.ChunkCacheDirectory!, $"{Hash:X16}_{Guid}.chunk"); + if (File.Exists(cachePath)) + { + CachePath = cachePath; + using var fileHandle = File.OpenHandle(CachePath); + return await RandomAccess.ReadAsync(fileHandle, buffer.AsMemory(offset, count), chunkPartOffset, cancellationToken).ConfigureAwait(false); + } + } + + byte[]? poolBuffer = null; + byte[]? uncompressPoolBuffer = null; + + try + { + var uri = GetUri(manifest); + using var res = await manifest.Options.Client!.GetAsync(uri, cancellationToken).ConfigureAwait(false); + EnsureSuccessStatusCode(res, uri); + var poolBufferSize = res.Content.Headers.ContentLength ?? manifest.Options.ChunkDownloadBufferSize; + poolBuffer = ArrayPool.Shared.Rent((int)poolBufferSize); + var destMs = new MemoryStream(poolBuffer, 0, poolBuffer.Length, true, true); + await res.Content.CopyToAsync(destMs, cancellationToken).ConfigureAwait(false); + var responseSize = (int)destMs.Length; + + var header = FChunkHeader.Parse(new ManifestData(poolBuffer, 0, responseSize)); + + if (header.StoredAs == EChunkStorageFlags.None) + { + Unsafe.CopyBlockUnaligned(ref buffer[offset], ref poolBuffer[header.HeaderSize + chunkPartOffset], (uint)count); + if (!shouldCache) + return count; + using (var fileHandle = File.OpenHandle(cachePath!, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, header.DataSizeCompressed)) + { + await RandomAccess.WriteAsync(fileHandle, new ReadOnlyMemory(poolBuffer, header.HeaderSize, header.DataSizeCompressed), 0, cancellationToken).ConfigureAwait(false); + RandomAccess.FlushToDisk(fileHandle); + } + CachePath = cachePath; + return count; + } + + if (header.StoredAs.HasFlag(EChunkStorageFlags.Encrypted)) + throw new NotSupportedException("Encrypted chunks are not supported"); + if (!header.StoredAs.HasFlag(EChunkStorageFlags.Compressed)) + throw new UnreachableException("Unknown/new chunk ChunkStorageFlag"); + if (manifest.Options.Decompressor is null) + throw new InvalidOperationException("Data is compressed and decompressor delegate was null"); + + // cant seek for uncompression + uncompressPoolBuffer = ArrayPool.Shared.Rent(header.DataSizeUncompressed); + + var result = manifest.Options.Decompressor.Invoke( + manifest.Options.DecompressorState, + poolBuffer, header.HeaderSize, header.DataSizeCompressed, + uncompressPoolBuffer, 0, header.DataSizeUncompressed); + if (!result) + throw new FileLoadException("Failed to uncompress data"); + + Unsafe.CopyBlockUnaligned(ref buffer[offset], ref uncompressPoolBuffer[chunkPartOffset], (uint)count); + if (!shouldCache) + return count; + using (var fileHandle = File.OpenHandle(cachePath!, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, header.DataSizeUncompressed)) + { + await RandomAccess.WriteAsync(fileHandle, new ReadOnlyMemory(uncompressPoolBuffer, 0, header.DataSizeUncompressed), 0, cancellationToken).ConfigureAwait(false); + RandomAccess.FlushToDisk(fileHandle); + } + CachePath = cachePath; + return count; + } + finally + { + if (poolBuffer is not null) + ArrayPool.Shared.Return(poolBuffer); + if (uncompressPoolBuffer is not null) + ArrayPool.Shared.Return(uncompressPoolBuffer); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void EnsureSuccessStatusCode(HttpResponseMessage res, Uri uri) + { + try + { + res.EnsureSuccessStatusCode(); + } + catch (HttpRequestException ex) + { + ex.Data.Add("Uri", uri); + ex.Data.Add("Headers", res.Headers); + throw; + } + } + + // ReSharper disable once UseSymbolAlias + internal static void Test_Zlibng(byte[] uncompressPoolBuffer, byte[] chunkBuffer, object zlibng, ManifestParseOptions.DecompressDelegate zlibngUncompress) + { + var header = FChunkHeader.Parse(chunkBuffer); + + var result = zlibngUncompress( + zlibng, + chunkBuffer, header.HeaderSize, header.DataSizeCompressed, + uncompressPoolBuffer, 0, header.DataSizeUncompressed); + + if (!result) + throw new FileLoadException("Failed to uncompress chunk data"); + } + + // ReSharper disable once UseSymbolAlias + internal static void Test_ZlibStream(byte[] uncompressPoolBuffer, byte[] chunkBuffer) + { + var header = FChunkHeader.Parse(chunkBuffer); + + var result = ManifestZlibStreamDecompressor.Decompress( + null, + chunkBuffer, header.HeaderSize, header.DataSizeCompressed, + uncompressPoolBuffer, 0, header.DataSizeUncompressed); + + if (!result) + throw new FileLoadException("Failed to uncompress chunk data"); + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkPart.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkPart.cs new file mode 100644 index 0000000..a54a9af --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkPart.cs @@ -0,0 +1,49 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FChunkPart struct +/// +public readonly struct FChunkPart +{ + /// + /// The GUID of the chunk containing this part. + /// + public FGuid Guid { get; } + /// + /// The offset of the first byte into the chunk. + /// + public uint32 Offset { get; } + /// + /// The size of this part. + /// + public uint32 Size { get; } + + internal FChunkPart(FGuid guid, uint32 offset, uint32 size) + { + Guid = guid; + Offset = offset; + Size = size; + } + + internal FChunkPart(ref ManifestReader reader) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + + Guid = reader.Read(); + Offset = reader.Read(); + Size = reader.Read(); + + reader.Position = startPos + dataSize; + } + +#if NET9_0_OR_GREATER + internal static FChunkPart Read(ref ManifestReader reader) => new(ref reader); +#else + internal static FChunkPart Read(GenericReader.IGenericReader genericReader) + { + var reader = (ManifestReader)genericReader; + return new FChunkPart(ref reader); + } +#endif +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FCustomField.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FCustomField.cs new file mode 100644 index 0000000..e015187 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FCustomField.cs @@ -0,0 +1,49 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FCustomField struct +/// +public sealed class FCustomField +{ + /// + /// Field name + /// + public string Name { get; internal set; } = ""; + /// + /// Field value + /// + public string Value { get; internal set; } = ""; + + internal FCustomField() { } + + internal static FCustomField[] ReadCustomFields(ref ManifestReader reader) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + var elementCount = reader.Read(); + + var fields = new FCustomField[elementCount]; + var fieldsSpan = fields.AsSpan(); + + if (dataVersion >= EChunkDataListVersion.Original) + { + for (var i = 0; i < elementCount; i++) + { + var field = new FCustomField(); + field.Name = reader.ReadFString(); + fieldsSpan[i] = field; + } + for (var i = 0; i < elementCount; i++) + fieldsSpan[i].Value = reader.ReadFString(); + } + else + { + var defaultField = new FCustomField(); + fieldsSpan.Fill(defaultField); + } + + reader.Position = startPos + dataSize; + return fields; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifest.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifest.cs new file mode 100644 index 0000000..b213dc2 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifest.cs @@ -0,0 +1,164 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FFileManifest struct +/// +public sealed class FFileManifest : IComparable, IComparable +{ + /// + /// The build relative filename. + /// + public string FileName { get; internal set; } = ""; + /// + /// Whether this is a symlink to another file. + /// + public string SymlinkTarget { get; internal set; } = ""; + /// + /// The file SHA1. + /// + public FSHAHash FileHash { get; internal set; } + /// + /// The flags for this file. + /// + public EFileMetaFlags FileMetaFlags { get; internal set; } + /// + /// The install tags for this file. + /// + public IReadOnlyList InstallTags { get; internal set; } = []; + /// + /// The list of chunk parts to stitch. + /// + public IReadOnlyList ChunkParts => ChunkPartsArray; + internal FChunkPart[] ChunkPartsArray = []; + /// + /// The size of this file. + /// + public int64 FileSize { get; internal set; } + /// + /// The mime type. + /// + public string MimeType { get; internal set; } = ""; + + internal FBuildPatchAppManifest Manifest { get; set; } = null!; + + internal FFileManifest() { } + + internal static FFileManifest[] ReadFileDataList(ref ManifestReader reader, FBuildPatchAppManifest manifest) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + var elementCount = reader.Read(); + + var files = new FFileManifest[elementCount]; + var filesSpan = files.AsSpan(); + + if (dataVersion >= EFileManifestListVersion.Original) + { + for (var i = 0; i < elementCount; i++) + { + var file = new FFileManifest(); + file.FileName = reader.ReadFString(); + filesSpan[i] = file; + } + for (var i = 0; i < elementCount; i++) + filesSpan[i].SymlinkTarget = reader.ReadFString(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].FileHash = reader.Read(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].FileMetaFlags = reader.Read(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].InstallTags = reader.ReadFStringArray(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].ChunkPartsArray = reader.ReadArray(FChunkPart.Read); + + // not to be found in UE, maybe fn specific? + if (dataVersion >= (EFileManifestListVersion)2) + { + for (var i = 0; i < elementCount; i++) // TArray + { + var a = reader.Read(); + reader.Position += a * 16; + } + for (var i = 0; i < elementCount; i++) + filesSpan[i]!.MimeType = reader.ReadFString(); + for (var i = 0; i < elementCount; i++) // Unknown + reader.Position += 32; + } + + // FileDataList.OnPostLoad(); + { + Array.Sort(files); + for (var i = 0; i < elementCount; i++) + { + var file = filesSpan[i]; + file.Manifest = manifest; + foreach (var chunkPart in file.ChunkPartsArray.AsSpan()) + { + file.FileSize += chunkPart.Size; + } + } + } + } + else + { + var defaultFile = new FFileManifest(); + filesSpan.Fill(defaultFile); + } + + reader.Position = startPos + dataSize; + return files; + } + + /// + /// Creates a read-only stream to read filedata from. + /// + public FFileManifestStream GetStream() => new(this, Manifest.Options.CacheChunksAsIs); + + /// + /// Creates a read-only stream to read filedata from. + /// + /// Whether or not to cache the chunks 1:1 as they were downloaded. + public FFileManifestStream GetStream(bool cacheAsIs) => new(this, cacheAsIs); + + + /// + public int CompareTo(FFileManifest? other) + { + if (ReferenceEquals(this, other)) return 0; + if (ReferenceEquals(null, other)) return 1; + return string.Compare(FileName, other.FileName, StringComparison.Ordinal); + } + + /// + public int CompareTo(object? obj) + { + if (ReferenceEquals(null, obj)) return 1; + if (ReferenceEquals(this, obj)) return 0; + return obj is FFileManifest other ? CompareTo(other) : throw new ArgumentException($"Object must be of type {nameof(FFileManifest)}"); + } + + /// + public static bool operator <(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) < 0; + } + + /// + public static bool operator >(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) > 0; + } + + /// + public static bool operator <=(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) <= 0; + } + + /// + public static bool operator >=(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) >= 0; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifestStream.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifestStream.cs new file mode 100644 index 0000000..85c1958 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifestStream.cs @@ -0,0 +1,662 @@ +using Microsoft.Win32.SafeHandles; +using OffiUtils; +using System.Buffers; +using System.Collections; +using System.Runtime.CompilerServices; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; +// ReSharper disable UseSymbolAlias + +/// +/// A stream representing a +/// +public sealed class FFileManifestStream : RandomAccessStream +{ + private readonly FFileManifest _fileManifest; + private readonly bool _cacheAsIs; + + /// Always + public override bool CanRead => true; + /// Always + public override bool CanSeek => true; + /// Always + public override bool CanWrite => false; + /// Gets the length/size of the stream + public override long Length => _fileManifest.FileSize; + + private long _position; + + /// Gets or sets the current position within the stream. + public override long Position + { + get => _position; + set + { + if ((ulong)value > (ulong)Length) + throw new ArgumentOutOfRangeException(nameof(Position), value, "Value is negative or exceeds the stream's length"); + + _position = value; + } + } + + /// Gets the file name of the represented by this stream + public string FileName => _fileManifest.FileName; + + internal FFileManifestStream(FFileManifest fileManifest, bool cacheAsIs) + { + if (string.IsNullOrEmpty(fileManifest.Manifest.Options.ChunkBaseUrl)) + throw new ArgumentException("Missing ChunkBaseUrl"); + if (fileManifest.Manifest.Meta.bIsFileData) + throw new NotSupportedException("File-data manifests are not supported"); + + _fileManifest = fileManifest; + _cacheAsIs = cacheAsIs; + } + + /// + /// Asynchronously saves the current stream to another stream. + /// + /// The destination stream. + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveToAsync(Stream destination, Action? progressCallback, + object? userState = default, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + if (destination is MemoryStream { Position: 0 } ms) + { + ms.Capacity = (int)Length; + if (ms.TryGetBuffer(out var buffer)) + { + await SaveBytesAsync(buffer.Array!, progressCallback, userState, maxDegreeOfParallelism, cancellationToken).ConfigureAwait(false); + ms.Position = (int)Length; + return; + } + } + + // TODO: make concurrent + + var downloadState = new DownloadState(null!, _fileManifest, Length, userState, progressCallback); + + if (_cacheAsIs) + { + var poolBuffer = ArrayPool.Shared.Rent(_fileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + foreach (var fileChunkPart in _fileManifest.ChunkPartsArray) + { + var chunk = _fileManifest.Manifest.Chunks[fileChunkPart.Guid]; + await chunk.ReadDataAsIsAsync(poolBuffer, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + await destination.WriteAsync(new ReadOnlyMemory(poolBuffer, (int)fileChunkPart.Offset, (int)fileChunkPart.Size), + cancellationToken).ConfigureAwait(false); + downloadState.OnBytesWritten(fileChunkPart.Size); + } + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + else + { + var poolBuffer = ArrayPool.Shared.Rent(_fileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + foreach (var fileChunkPart in _fileManifest.ChunkPartsArray) + { + var chunk = _fileManifest.Manifest.Chunks[fileChunkPart.Guid]; + await chunk.ReadDataAsync(poolBuffer, 0, (int)fileChunkPart.Size, + (int)fileChunkPart.Offset, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + await destination.WriteAsync(new ReadOnlyMemory(poolBuffer, 0, (int)fileChunkPart.Size), + cancellationToken).ConfigureAwait(false); + downloadState.OnBytesWritten(fileChunkPart.Size); + } + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public Task SaveToAsync(Stream destination, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + return SaveToAsync(destination, null, 0, maxDegreeOfParallelism, cancellationToken); + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The destination buffer. + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveBytesAsync(byte[] destination, Action? progressCallback, + object? userState = default, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(destination.Length, Length); + + var downloadState = new DownloadState(destination, _fileManifest, Length, userState, progressCallback); + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = maxDegreeOfParallelism ?? Environment.ProcessorCount, + CancellationToken = cancellationToken + }; + + if (_cacheAsIs) + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsIsAsync).ConfigureAwait(false); + else + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsync).ConfigureAwait(false); + + return; + + static async ValueTask SaveAsync(ChunkWithOffset tuple, CancellationToken token) + { + await tuple.Chunk.ReadDataAsync(tuple.State.Destination, (int)tuple.Offset, (int)tuple.ChunkPartSize, + (int)tuple.ChunkPartOffset, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + + static async ValueTask SaveAsIsAsync(ChunkWithOffset tuple, CancellationToken token) + { + var poolBuffer = ArrayPool.Shared.Rent(tuple.State.FileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + await tuple.Chunk.ReadDataAsIsAsync(poolBuffer, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + Unsafe.CopyBlockUnaligned(ref tuple.State.Destination[tuple.Offset], + ref poolBuffer[tuple.ChunkPartOffset], tuple.ChunkPartSize); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The destination buffer. + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public Task SaveBytesAsync(byte[] destination, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + return SaveBytesAsync(destination, null, 0, maxDegreeOfParallelism, cancellationToken); + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveBytesAsync(Action progressCallback, object? userState = default, + int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + var destination = new byte[Length]; + await SaveBytesAsync(destination, progressCallback, userState, maxDegreeOfParallelism, cancellationToken).ConfigureAwait(false); + return destination; + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveBytesAsync(int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + var destination = new byte[Length]; + await SaveBytesAsync(destination, maxDegreeOfParallelism, cancellationToken).ConfigureAwait(false); + return destination; + } + + /// + /// Asynchronously saves the current stream to a file. + /// + /// The path of the destination file. + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveFileAsync(string path, Action? progressCallback, object? userState = null, + int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + using var destination = File.OpenHandle(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None, FileOptions.Asynchronous, Length); + var downloadState = new DownloadState(destination, _fileManifest, Length, userState, progressCallback); + + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = maxDegreeOfParallelism ?? Environment.ProcessorCount, + CancellationToken = cancellationToken + }; + + if (_cacheAsIs) + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsIsAsync).ConfigureAwait(false); + else + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsync).ConfigureAwait(false); + + RandomAccess.FlushToDisk(destination); + return; + + static async ValueTask SaveAsync(ChunkWithOffset tuple, CancellationToken token) + { + var poolBuffer = ArrayPool.Shared.Rent((int)tuple.ChunkPartSize); + + try + { + await tuple.Chunk.ReadDataAsync(poolBuffer, 0, (int)tuple.ChunkPartSize, + (int)tuple.ChunkPartOffset, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + await RandomAccess.WriteAsync(tuple.State.Destination, + new ReadOnlyMemory(poolBuffer, 0, (int)tuple.ChunkPartSize), + tuple.Offset, token).ConfigureAwait(false); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + + static async ValueTask SaveAsIsAsync(ChunkWithOffset tuple, CancellationToken token) + { + var poolBuffer = ArrayPool.Shared.Rent(tuple.State.FileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + await tuple.Chunk.ReadDataAsIsAsync(poolBuffer, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + await RandomAccess.WriteAsync(tuple.State.Destination, + new ReadOnlyMemory(poolBuffer, (int)tuple.ChunkPartOffset, (int)tuple.ChunkPartSize), + tuple.Offset, token).ConfigureAwait(false); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + } + + /// + public Task SaveFileAsync(string path, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + return SaveFileAsync(path, null, 0, maxDegreeOfParallelism, cancellationToken); + } + + /// + /// Reads a sequence of bytes from the current stream. + /// + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The total number of bytes written into the . + public override int Read(byte[] buffer, int offset, int count) + { + return ReadAsync(buffer, offset, count, CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Asynchronously reads a sequence of bytes from the the current stream. + /// + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The token to monitor for cancellation requests. + /// The total number of bytes written into the . + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + return await Task.FromCanceled(cancellationToken).ConfigureAwait(false); + + var bytesRead = await ReadAtAsync(_position, buffer, offset, count, cancellationToken).ConfigureAwait(false); + _position += bytesRead; + return bytesRead; + } + + /// + /// Asynchronously reads a sequence of bytes from the current stream. + /// + /// The region of memory to write the data into. + /// The token to monitor for cancellation requests. + /// The total number of bytes written into the . + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var bytesRead = await ReadAtAsync(_position, buffer, cancellationToken).ConfigureAwait(false); + _position += bytesRead; + return bytesRead; + } + + /// + /// Reads a sequence of bytes from the given of the current stream. + /// + /// The position to begin reading from. + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The total number of bytes written into the . + public override int ReadAt(long position, byte[] buffer, int offset, int count) + { + return ReadAtAsync(position, buffer, offset, count, CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Asynchronously reads a sequence of bytes from the given of the current stream. + /// + /// The position to begin reading from. + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The token to monitor for cancellation requests. + /// The total number of bytes written into the . + public override async Task ReadAtAsync(long position, byte[] buffer, int offset, int count, CancellationToken cancellationToken = default) + { + var (i, startPos) = GetChunkIndex(position); + if (i == -1) + return 0; + + var bytesRead = 0u; + + if (_cacheAsIs) + { + var poolBuffer = ArrayPool.Shared.Rent(_fileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + while (i < _fileManifest.ChunkPartsArray.Length) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + var chunk = _fileManifest.Manifest.Chunks[chunkPart.Guid]; + + await chunk.ReadDataAsIsAsync(poolBuffer, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + + var chunkOffset = chunkPart.Offset + startPos; + var chunkBytes = chunkPart.Size - startPos; + var bytesLeft = (uint)count - bytesRead; + + if (bytesLeft <= chunkBytes) + { + Unsafe.CopyBlockUnaligned(ref buffer[bytesRead + offset], ref poolBuffer[chunkOffset], bytesLeft); + bytesRead += bytesLeft; + break; + } + + Unsafe.CopyBlockUnaligned(ref buffer[bytesRead + offset], ref poolBuffer[chunkOffset], chunkBytes); + bytesRead += chunkBytes; + startPos = 0; + + ++i; + } + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + else + { + while (i < _fileManifest.ChunkPartsArray.Length) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + var chunk = _fileManifest.Manifest.Chunks[chunkPart.Guid]; + + var chunkOffset = (int)(chunkPart.Offset + startPos); + var chunkBytes = (int)(chunkPart.Size - startPos); + var bytesLeft = count - (int)bytesRead; + + if (bytesLeft <= chunkBytes) + { + await chunk.ReadDataAsync(buffer, (int)bytesRead + offset, bytesLeft, chunkOffset, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + bytesRead += (uint)bytesLeft; + break; + } + + await chunk.ReadDataAsync(buffer, (int)bytesRead + offset, chunkBytes, chunkOffset, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + bytesRead += (uint)chunkBytes; + startPos = 0; + + ++i; + } + } + + return (int)bytesRead; + } + + private long _lastChunkPartPosition; + private uint _lastChunkPartSize; + private int _lastChunkPartIndex; + + private (int Index, uint ChunkPos) GetChunkIndexNew(long position) + { + lock (_fileManifest) + { + var maxPosition = _lastChunkPartPosition + _lastChunkPartSize; + if (maxPosition < position && position >= _lastChunkPartPosition) + { + return (_lastChunkPartIndex, (uint)(_lastChunkPartPosition - position)); + } + + var chunkPartPosition = 0L; + + for (var i = 0; i < _fileManifest.ChunkPartsArray.Length; i++) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + + if (chunkPartPosition >= position) + { + _lastChunkPartPosition = chunkPartPosition; + _lastChunkPartSize = chunkPart.Size; + _lastChunkPartIndex = i; + return (i, (uint)(chunkPartPosition - position)); + } + + chunkPartPosition += chunkPart.Size; + } + + return (-1, 0); + } + } + + private (int Index, uint ChunkPos) GetChunkIndex(long position) + { + for (var i = 0; i < _fileManifest.ChunkPartsArray.Length; i++) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + + if (position < chunkPart.Size) + return (i, (uint)position); + + position -= chunkPart.Size; + } + + return (-1, 0); + } + + /// Sets the position within the current stream to the specified value. + /// The new position within the stream. This is relative to the parameter, and can be positive or negative. + /// A value of type , which acts as the seek reference point. + /// The new position within the stream, calculated by combining the initial reference point and the offset. + /// There is an invalid . + public override long Seek(long offset, SeekOrigin loc) + { + Position = loc switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => offset + _position, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentException("Invalid loc", nameof(loc)) + }; + return _position; + } + + /// Not supported + public override void SetLength(long value) + => throw new NotSupportedException(); + /// Not supported + public override void Write(byte[] buffer, int offset, int count) + => throw new NotSupportedException(); + /// Not supported + public override void Flush() + => throw new NotSupportedException(); +} + +/// +/// Event for save progress +/// +public sealed class SaveProgressChangedEventArgs : EventArgs +{ + internal SaveProgressChangedEventArgs(object? userState, long bytesSaved, long totalBytesToSave, int progressPercentage) + { + UserState = userState; + BytesSaved = bytesSaved; + TotalBytesToSave = totalBytesToSave; + ProgressPercentage = progressPercentage; + } + + /// + public object? UserState { get; } + /// + public long BytesSaved { get; } + /// + public long TotalBytesToSave { get; } + /// + public int ProgressPercentage { get; } +} + +internal sealed class DownloadState : IEnumerable>, IEnumerator> + where TDestination : class +{ + public readonly TDestination Destination; + public readonly FFileManifest FileManifest; + + private readonly LockObject? _lock; + private readonly object? _userState; + private readonly Action? _callback; + private readonly long _totalBytesToSave; + private long _bytesSaved; + private int _lastProgress; + + // IEnumerable & IEnumerator + private long _offset; + private long _lastSize; + private int _chunkpartIndex; + + public DownloadState(TDestination destination, FFileManifest fileManifest, long totalBytesToSave, object? userState, Action? callback) + { + Reset(); + Destination = destination; + FileManifest = fileManifest; + + if (callback is null) return; + _lock = new LockObject(); + _userState = userState; + _callback = callback; + _totalBytesToSave = totalBytesToSave; + } + + public void OnBytesWritten(long amount) + { + if (_callback is null) + return; + + lock (_lock!) + { + _bytesSaved += amount; + var progress = (int)MathF.Truncate((float)_bytesSaved / _totalBytesToSave * 100f); + if (progress != _lastProgress) + { + _lastProgress = progress; + var eventArgs = new SaveProgressChangedEventArgs(_userState, _bytesSaved, _totalBytesToSave, progress); + _callback(eventArgs); + } + } + } + + // IEnumerable + + public IEnumerator> GetEnumerator() => this; + IEnumerator IEnumerable.GetEnumerator() => this; + + // IEnumerator + + public bool MoveNext() + { + _chunkpartIndex++; + if (_chunkpartIndex >= FileManifest.ChunkPartsArray.Length) + return false; + + _offset += _lastSize; + var chunkPart = FileManifest.ChunkPartsArray[_chunkpartIndex]; + _lastSize = chunkPart.Size; + return true; + } + + public void Reset() + { + _offset = 0; + _lastSize = 0; + _chunkpartIndex = -1; + } + + public ChunkWithOffset Current + { + get + { + var chunkPart = FileManifest.ChunkPartsArray[_chunkpartIndex]; + var chunk = FileManifest.Manifest.Chunks[chunkPart.Guid]; + return new ChunkWithOffset(this, chunk, chunkPart.Offset, chunkPart.Size, _offset); + } + } + + object IEnumerator.Current => Current; + + public void Dispose() { } +} + +internal readonly struct ChunkWithOffset where TDestination : class +{ + public readonly DownloadState State; + public readonly FChunkInfo Chunk; + public readonly uint ChunkPartOffset; + public readonly uint ChunkPartSize; + public readonly long Offset; + + public ChunkWithOffset(DownloadState state, FChunkInfo chunk, uint chunkPartOffset, uint chunkPartSize, long offset) + { + State = state; + Chunk = chunk; + ChunkPartOffset = chunkPartOffset; + ChunkPartSize = chunkPartSize; + Offset = offset; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FGuid.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FGuid.cs new file mode 100644 index 0000000..1ca9a4d --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FGuid.cs @@ -0,0 +1,196 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Unicode; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FGuid struct +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public readonly struct FGuid : IEquatable, ISpanFormattable, IUtf8SpanFormattable +{ + /// + /// The size of the FGuid/struct. + /// + public const int Size = sizeof(uint32) * 4; + + private readonly uint32 A; + private readonly uint32 B; + private readonly uint32 C; + private readonly uint32 D; + + /// + /// Creates a hex string of the guid. + /// + public string GetHexString(bool upperCase = true) => upperCase + ? $"{A:X8}{B:X8}{C:X8}{D:X8}" + : $"{A:x8}{B:x8}{C:x8}{D:x8}"; + + /// + /// Creates a string of the guid. + /// + public string GetGuidString() => $"{A:x8}-{B >> 16:x4}-{B & 0xffff:x4}-{C >> 16:x4}-{C & 0xffff:x4}{D:x8}"; + + /// + /// Creates a FGuid from values. + /// + /// A value. + /// B value. + /// C value. + /// D value. + public FGuid(uint32 a, uint32 b, uint32 c, uint32 d) + { + A = a; + B = b; + C = c; + D = d; + } + + /// + /// Parses a FGuid from a string. + /// + /// The FGuid string. + public FGuid(string guid) : this(guid.AsSpan()) { } + + /// + /// Parses a FGuid from a string. + /// + /// The FGuid string. + public FGuid(ReadOnlySpan guid) + { + if (guid.Length != 32) + throw new ArgumentOutOfRangeException(nameof(guid), "guid has to be 32 characters long, other parsing is not implemented"); + A = uint32.Parse(guid[..8], NumberStyles.AllowHexSpecifier); + B = uint32.Parse(guid[8..16], NumberStyles.AllowHexSpecifier); + C = uint32.Parse(guid[16..24], NumberStyles.AllowHexSpecifier); + D = uint32.Parse(guid[24..32], NumberStyles.AllowHexSpecifier); + } + + /// + /// Parses a FGuid from a UTF8 string. + /// + /// The UTF8 FGuid string. + public FGuid(ReadOnlySpan utf8Guid) + { + if (utf8Guid.Length != 32) + throw new ArgumentOutOfRangeException(nameof(utf8Guid), "guid has to be 32 characters long, other parsing is not implemented"); + A = uint32.Parse(utf8Guid[..8], NumberStyles.AllowHexSpecifier); + B = uint32.Parse(utf8Guid[8..16], NumberStyles.AllowHexSpecifier); + C = uint32.Parse(utf8Guid[16..24], NumberStyles.AllowHexSpecifier); + D = uint32.Parse(utf8Guid[24..32], NumberStyles.AllowHexSpecifier); + } + + /// + /// Creates a random FGuid. + /// + public static FGuid Random() + { + Unsafe.SkipInit(out FGuid result); + RandomNumberGenerator.Fill(result.GetSpan()); + return result; + } + + /// + /// Checks for validity. + /// + public bool IsValid() => (A | B | C | D) != 0; + + /// + /// Gets the data of the guid. + /// + public ReadOnlySpan AsSpan() => + MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref Unsafe.AsRef(in A)), Size); + + /// + /// Gets the values of the guid. + /// + public ReadOnlySpan AsIntSpan() => + MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in A), 4); + + internal Span GetSpan() => + MemoryMarshal.CreateSpan(ref Unsafe.As(ref Unsafe.AsRef(in A)), Size); + + /// + public bool Equals(FGuid other) + { + return A == other.A && B == other.B && C == other.C && D == other.D; + } + + /// + public override bool Equals(object? obj) + { + return obj is FGuid other && Equals(other); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(A, B, C, D); + } + + /// + public static bool operator ==(FGuid left, FGuid right) + { + return left.Equals(right); + } + + /// + public static bool operator !=(FGuid left, FGuid right) + { + return !left.Equals(right); + } + + /// + public override string ToString() => GetHexString(); + + /// + public string ToString(string? format, IFormatProvider? formatProvider) + { + FormattableString formattable = $"{A:X8}{B:X8}{C:X8}{D:X8}"; + return formattable.ToString(formatProvider); + } + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + return destination.TryWrite(provider, $"{A:X8}{B:X8}{C:X8}{D:X8}", out charsWritten); + } + + /// + public bool TryFormat(Span destination, out int bytesWritten, ReadOnlySpan format, IFormatProvider? provider) + { + return Utf8.TryWrite(destination, provider, $"{A:X8}{B:X8}{C:X8}{D:X8}", out bytesWritten); + } +} + +/// +/// Converts from and to JSON. +/// +public sealed class FGuidConverter : JsonConverter +{ + /// + public override FGuid Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.ValueSpan.IsEmpty) return default; + return new FGuid(reader.ValueSpan); + } + + /// + public override void Write(Utf8JsonWriter writer, FGuid value, JsonSerializerOptions options) + { + Span guidUtf8 = stackalloc byte[FGuid.Size * 2]; + + if (value.TryFormat(guidUtf8, out _, default, null)) + { + writer.WriteStringValue(guidUtf8); + return; + } + + writer.WriteStringValue(string.Empty); + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestHeader.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestHeader.cs new file mode 100644 index 0000000..afcc577 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestHeader.cs @@ -0,0 +1,64 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal class FManifestHeader +{ + public const uint32 Magic = 0x44BEC00C; + + /// + /// The version of this header and manifest data format, driven by the feature level. + /// + public readonly EFeatureLevel Version; + /// + /// The size of this header. + /// + public readonly int32 HeaderSize; + /// + /// The size of this data compressed. + /// + public readonly int32 DataSizeCompressed; + /// + /// The size of this data uncompressed. + /// + public readonly int32 DataSizeUncompressed; + /// + /// How the chunk data is stored. + /// + public readonly EManifestStorageFlags StoredAs; + /// + /// The SHA1 hash for the manifest data that follows. + /// + public readonly FSHAHash SHAHash; + + internal FManifestHeader(ref ManifestReader reader) + { + var magic = reader.Read(); + if (magic != Magic) + throw new FileLoadException($"Invalid manifest header magic: 0x{magic:X}"); + + HeaderSize = reader.Read(); + DataSizeUncompressed = reader.Read(); + DataSizeCompressed = reader.Read(); + SHAHash = reader.Read(); + StoredAs = reader.Read(); + Version = HeaderSize > ManifestHeaderVersionSizes[(int32)EFeatureLevel.Original] + ? reader.Read() + : EFeatureLevel.StoredAsCompressedUClass; + + reader.SetPosition(HeaderSize); + } + + // The constant minimum sizes for each version of a header struct. Must be updated. + // If new member variables are added the version MUST be bumped and handled properly here, + // and these values must never change. + private static readonly uint32[] ManifestHeaderVersionSizes = + [ + // EFeatureLevel::Original is 37B (32b Magic, 32b HeaderSize, 32b DataSizeUncompressed, 32b DataSizeCompressed, 160b SHA1, 8b StoredAs) + // This remained the same all up to including EFeatureLevel::StoresPrerequisiteIds. + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + // EFeatureLevel::StoredAsBinaryData is 41B, (296b Original, 32b Version). + // This remained the same all up to including EFeatureLevel::UsesBuildTimeGeneratedBuildId. + 41, 41, 41, 41, 41, + // Undocumented, but the latest version is still 41B + 41, 41, 41 + ]; +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestMeta.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestMeta.cs new file mode 100644 index 0000000..c428183 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestMeta.cs @@ -0,0 +1,107 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FManifestMeta struct +/// +public sealed class FManifestMeta +{ + /// + /// The feature level support this build was created with, regardless of the serialised format. + /// + public EFeatureLevel FeatureLevel { get; internal set; } = EFeatureLevel.Invalid; + /// + /// Whether this is a legacy 'nochunks' build. + /// + public bool bIsFileData { get; internal set; } + /// + /// The app id provided at generation. + /// + public uint32 AppID { get; internal set; } + /// + /// The app name string provided at generation. + /// + public string AppName { get; internal set; } = ""; + /// + /// The build version string provided at generation. + /// + public string BuildVersion { get; internal set; } = ""; + /// + /// The file in this manifest designated the application executable of the build. + /// + public string LaunchExe { get; internal set; } = ""; + /// + /// The command line required when launching the application executable. + /// + public string LaunchCommand { get; internal set; } = ""; + /// + /// The set of prerequisite ids for dependencies that this build's prerequisite installer will apply. + /// + public string[] PrereqIds { get; internal set; } = []; + /// + /// A display string for the prerequisite provided at generation. + /// + public string PrereqName { get; internal set; } = ""; + /// + /// The file in this manifest designated the launch executable of the prerequisite installer. + /// + public string PrereqPath { get; internal set; } = ""; + /// + /// The command line required when launching the prerequisite installer. + /// + public string PrereqArgs { get; internal set; } = ""; + /// + /// A unique build id generated at original chunking time to identify an exact build. + /// + public string BuildId { get; internal set; } = ""; + + /// + /// Undocumented + /// + public string UninstallExe { get; internal set; } = ""; + /// + /// Undocumented + /// + public string UninstallCommand { get; internal set; } = ""; + + internal FManifestMeta() { } + internal FManifestMeta(ref ManifestReader reader) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + + if (dataVersion >= EManifestMetaVersion.Original) + { + FeatureLevel = reader.Read(); + bIsFileData = reader.Read() == 1; + AppID = reader.Read(); + AppName = reader.ReadFString(); + BuildVersion = reader.ReadFString(); + LaunchExe = reader.ReadFString(); + LaunchCommand = reader.ReadFString(); + PrereqIds = reader.ReadFStringArray(); + PrereqName = reader.ReadFString(); + PrereqPath = reader.ReadFString(); + PrereqArgs = reader.ReadFString(); + } + + BuildId = dataVersion >= EManifestMetaVersion.SerialisesBuildId + ? reader.ReadFString() + : GetBackwardsCompatibleBuildId(this); + + // not to be found in UE, maybe fn specific? + if (FeatureLevel > EFeatureLevel.UsesBuildTimeGeneratedBuildId) + { + UninstallExe = reader.ReadFString(); + UninstallCommand = reader.ReadFString(); + } + + reader.Position = startPos + dataSize; + } + + internal static string GetBackwardsCompatibleBuildId(in FManifestMeta meta) + { + // TODO: https://github.com/EpicGames/UnrealEngine/blob/a937fa584fbd6d69b7cf9c527907040c9dbf54fc/Engine/Source/Runtime/Online/BuildPatchServices/Private/BuildPatchUtil.cpp#L166 + return ""; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FSHAHash.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FSHAHash.cs new file mode 100644 index 0000000..b538b18 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FSHAHash.cs @@ -0,0 +1,163 @@ +using OffiUtils; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; +// ReSharper disable InconsistentNaming +// ReSharper disable UseSymbolAlias + +/// +/// UE FSHAHash struct +/// +[StructLayout(LayoutKind.Sequential, Pack = 1, Size = Size)] +public readonly struct FSHAHash : IEquatable, ISpanFormattable +{ + /// + /// The size of the hash/struct. + /// + public const int Size = 20; + + private readonly long Hash_00_07; + private readonly long Hash_08_15; + private readonly int Hash_16_19; + + /// + public bool Equals(FSHAHash other) + { + return Hash_00_07 == other.Hash_00_07 && Hash_08_15 == other.Hash_08_15 && Hash_16_19 == other.Hash_16_19; + } + + /// + public override bool Equals(object? obj) + { + return obj is FSHAHash other && Equals(other); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(Hash_00_07, Hash_08_15, Hash_16_19); + } + + /// + public static bool operator ==(FSHAHash left, FSHAHash right) + { + return left.Equals(right); + } + + /// + public static bool operator !=(FSHAHash left, FSHAHash right) + { + return !left.Equals(right); + } + + /// + /// Gets the data of the hash. + /// + /// Hash data in a read-only span + public ReadOnlySpan AsSpan() => + MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref Unsafe.AsRef(in Hash_00_07)), Size); + + internal Span GetSpan() => + MemoryMarshal.CreateSpan(ref Unsafe.As(ref Unsafe.AsRef(in Hash_00_07)), Size); + + /// + /// Computes the hash of data using the SHA1 algorithm. + /// + /// The data to hash + /// The computed hash + public static FSHAHash Compute(ReadOnlySpan source) + { + Unsafe.SkipInit(out FSHAHash hash); + SHA1.TryHashData(source, hash.GetSpan(), out _); + return hash; + } + + /// Returns a representation of the current instance. + /// The value of this , represented as a series of uppercase hexadecimal digits. + public override string ToString() => StringUtils.BytesToHexUpper(AsSpan()); + + /// Returns a representation of the current instance. + /// Whether or not to return an uppercase string. + /// The value of this , represented as a series of hexadecimal digits. + public string ToString(bool upperCase) => upperCase + ? StringUtils.BytesToHexUpper(AsSpan()) + : StringUtils.BytesToHexLower(AsSpan()); + + /// Returns a representation of the current instance, according to the provided format specifier. + /// A read-only span containing the character representing one of the following specifiers that indicates the exact format to use when interpreting input:
+ /// "x" or "X".
+ /// When is or empty, "X" is used. + /// + /// Unused, pass a null reference. + /// The value of this , represented as a series of hexadecimal digits in the specified format. + /// If an invalid format is used. + public string ToString(string? format, IFormatProvider? formatProvider) + { + if (format is null || format.Length == 0 || format == "X") + return StringUtils.BytesToHexUpper(AsSpan()); + if (format == "x") + return StringUtils.BytesToHexLower(AsSpan()); + throw new FormatException("the provided format is not valid"); + } + + /// + /// Tries to format the current instance into the provided character span. + /// + /// The span in which to write the as a span of characters. + /// When this method returns, contains the number of characters written into the span. + /// A read-only span containing the character representing one of the following specifiers that indicates the exact format to use when interpreting input:
+ /// "x" or "X".
+ /// When is empty, "X" is used. + /// + /// Unused, pass a null reference. + /// if the formatting was successful; otherwise, . + /// If an invalid format is used. + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + if (format.IsEmpty || format is "X") + return StringUtils.TryWriteBytesToHexUpper(AsSpan(), destination, out charsWritten); + if (format is "x") + return StringUtils.TryWriteBytesToHexLower(AsSpan(), destination, out charsWritten); + throw new FormatException("the provided format is not valid"); + } +} + +/// +/// Converts from and to JSON. +/// +public sealed class FSHAHashConverter : JsonConverter +{ + /// + public override FSHAHash Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + Unsafe.SkipInit(out FSHAHash result); + var resultSpan = result.GetSpan(); + var span = reader.ValueSpan; + + for (var i = 0; i < resultSpan.Length; i++) + { + resultSpan[i] = byte.Parse(span.Slice(i * 2, 2), NumberStyles.AllowHexSpecifier); + } + + return result; + } + + /// + public override void Write(Utf8JsonWriter writer, FSHAHash value, JsonSerializerOptions options) + { + Span hashUtf16 = stackalloc char[FSHAHash.Size * 2]; + + if (value.TryFormat(hashUtf16, out _, default, null)) + { + writer.WriteStringValue(hashUtf16); + return; + } + + writer.WriteStringValue(string.Empty); + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/TypeAliases.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/TypeAliases.cs new file mode 100644 index 0000000..afeadc0 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/TypeAliases.cs @@ -0,0 +1,5 @@ +global using int32 = System.Int32; +global using int64 = System.Int64; +global using uint32 = System.UInt32; +global using uint64 = System.UInt64; +global using uint8 = System.Byte; diff --git a/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs b/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs index 9f8e870..5d95dc5 100644 --- a/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs +++ b/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs @@ -20,7 +20,7 @@ public void TestTenLinesEach200_LF() var line = new string('a', 200); // a line with 200 characters var content = string.Join("\n", Enumerable.Repeat(line, 11)); // 11 such lines var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -28,7 +28,7 @@ public void TestTenLinesEach200_LF() // Assert Assert.AreEqual(10 * 200 + 10, sut.TotalBytesRead); // 10 newlines added between 10 lines - data.ForEach(d => Assert.AreEqual(200, d.Length)); + data.ForEach(d => Assert.AreEqual(200, d!.Length)); } [TestMethod] @@ -37,7 +37,7 @@ public void TestLineExactly1024_LF() // Arrange var line = new string('b', 1023); var stream = MockStream(line + "\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -45,7 +45,7 @@ public void TestLineExactly1024_LF() // Assert Assert.AreEqual(1024, sut.TotalBytesRead); - Assert.AreEqual(1023, data[0].Length); + Assert.AreEqual(1023, data[0]!.Length); } [TestMethod] @@ -56,7 +56,7 @@ public void TestLine1023And1025_LF() var line2 = new string('d', 1025); var content = $"{line1}\n{line2}\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -64,8 +64,8 @@ public void TestLine1023And1025_LF() // Assert Assert.AreEqual(1023 + 1025 + 2, sut.TotalBytesRead); // +2 for the two '\n' sequences - Assert.AreEqual(1023, data[0].Length); - Assert.AreEqual(1025, data[1].Length); + Assert.AreEqual(1023, data[0]!.Length); + Assert.AreEqual(1025, data[1]!.Length); } [TestMethod] @@ -75,7 +75,7 @@ public void TestTenLinesEach200_CRLF() var line = new string('a', 200); var content = string.Join("\r\n", Enumerable.Repeat(line, 11)); var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -83,7 +83,7 @@ public void TestTenLinesEach200_CRLF() // Assert Assert.AreEqual(10 * 200 + 10 * 2, sut.TotalBytesRead); - data.ForEach(d => Assert.AreEqual(200, d.Length)); + data.ForEach(d => Assert.AreEqual(200, d!.Length)); } [TestMethod] @@ -92,7 +92,7 @@ public void TestLineExactly1024_CRLF() // Arrange var line = new string('b', 1022); var stream = MockStream(line + "\r\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -100,7 +100,7 @@ public void TestLineExactly1024_CRLF() // Assert Assert.AreEqual(1024, sut.TotalBytesRead); - Assert.AreEqual(1022, data[0].Length); + Assert.AreEqual(1022, data[0]!.Length); } [TestMethod] @@ -109,7 +109,7 @@ public void TestLineExactly1025_CRLF() // Arrange var line = new string('b', 1023); var stream = MockStream(line + "\r\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -117,7 +117,7 @@ public void TestLineExactly1025_CRLF() // Assert Assert.AreEqual(1025, sut.TotalBytesRead); - Assert.AreEqual(1023, data[0].Length); + Assert.AreEqual(1023, data[0]!.Length); } [TestMethod] @@ -126,7 +126,7 @@ public void TestLineExactly1026_CRLF() // Arrange var line = new string('b', 1024); var stream = MockStream(line + "\r\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -134,7 +134,7 @@ public void TestLineExactly1026_CRLF() // Assert Assert.AreEqual(1026, sut.TotalBytesRead); - Assert.AreEqual(1024, data[0].Length); + Assert.AreEqual(1024, data[0]!.Length); } [TestMethod] @@ -145,7 +145,7 @@ public void TestLine1023And1025_CRLF() var line2 = new string('d', 1025); var content = $"{line1}\r\n{line2}\r\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -153,8 +153,8 @@ public void TestLine1023And1025_CRLF() // Assert Assert.AreEqual(1023 + 1025 + 4, sut.TotalBytesRead); - Assert.AreEqual(1023, data[0].Length); - Assert.AreEqual(1025, data[1].Length); + Assert.AreEqual(1023, data[0]!.Length); + Assert.AreEqual(1025, data[1]!.Length); } [TestMethod] @@ -164,7 +164,7 @@ public void When_InitialTotalBytesReadIsSetToSkipFirstLine_ThenShouldOnlyReadLas var initialTotalBytesRead = 4; var content = "abc\ndef\nghi\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = initialTotalBytesRead }; @@ -186,7 +186,7 @@ public void When_InitialTotalBytesReadIsGreaterThanStreamLength_PositionIsAdjust // Arrange var content = "line1\nline2\nline3\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = 500 }; @@ -210,7 +210,7 @@ public void When_DoubleNewLine_AtExactBufferLength_Works() var stringOfLength1024 = new string('a', 1023); var content = $"{stringOfLength1024}\n\nline2\nline3\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = 0 }; @@ -235,7 +235,7 @@ public void When_DoubleNewLine_AtExactBufferLength_CRLF_Works() var stringOfLength1024 = new string('a', 1023); var content = $"{stringOfLength1024}\n\r\nline2\nline3\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = 0 }; diff --git a/DeveLanCacheUI_Backend.Tests/UnitTest1.cs b/DeveLanCacheUI_Backend.Tests/UnitTest1.cs deleted file mode 100644 index ea40382..0000000 --- a/DeveLanCacheUI_Backend.Tests/UnitTest1.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace DeveLanCacheUI_Backend.Tests -{ - [TestClass] - public class UnitTest1 - { - [TestMethod] - public void TestMethod1() - { - } - } -} \ No newline at end of file diff --git a/DeveLanCacheUI_Backend.sln b/DeveLanCacheUI_Backend.sln index 6dcd8a1..788a2ba 100644 --- a/DeveLanCacheUI_Backend.sln +++ b/DeveLanCacheUI_Backend.sln @@ -7,6 +7,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeveLanCacheUI_Backend", "D EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeveLanCacheUI_Backend.Tests", "DeveLanCacheUI_Backend.Tests\DeveLanCacheUI_Backend.Tests.csproj", "{61B234FC-1790-4951-B763-9D93CA535DF8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeveLanCacheUI_Backend.EpicManifestParser", "DeveLanCacheUI_Backend.EpicManifestParser\DeveLanCacheUI_Backend.EpicManifestParser.csproj", "{8CBBAE99-5E56-41BA-BB81-66FD5E459631}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeveLanCacheUI_Backend.EpicManifestParser.Tests", "DeveLanCacheUI_Backend.EpicManifestParser.Tests\DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj", "{78517F5E-180D-41D8-B9EE-84B8F3811F92}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -21,6 +25,14 @@ Global {61B234FC-1790-4951-B763-9D93CA535DF8}.Debug|Any CPU.Build.0 = Debug|Any CPU {61B234FC-1790-4951-B763-9D93CA535DF8}.Release|Any CPU.ActiveCfg = Release|Any CPU {61B234FC-1790-4951-B763-9D93CA535DF8}.Release|Any CPU.Build.0 = Release|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Release|Any CPU.Build.0 = Release|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Debug|Any CPU.Build.0 = Debug|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Release|Any CPU.ActiveCfg = Release|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs b/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs index 9be7d43..cfca075 100644 --- a/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs +++ b/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs @@ -38,7 +38,7 @@ private async Task> GetFilteredBySkipAndCountInternal var excludedIps = _config.ExcludedClientIpsArray ?? Array.Empty(); IQueryable tmpResult = _dbContext.DownloadEvents; - + tmpResult = tmpResult.Where(t => !excludedIps.Contains(t.ClientIp)); if (filter != null) @@ -52,6 +52,8 @@ private async Task> GetFilteredBySkipAndCountInternal var query = from downloadEvent in pagedSubQuery join steamDepot in _dbContext.SteamDepots on downloadEvent.DownloadIdentifier equals steamDepot.SteamDepotId into steamDepotJoin from steamDepot in steamDepotJoin.DefaultIfEmpty() + join epicManifest in _dbContext.EpicManifests on downloadEvent.DownloadIdentifierString equals epicManifest.DownloadIdentifier into epicManifestJoin + from epicManifest in epicManifestJoin.DefaultIfEmpty() let steamManifest = (from sm in _dbContext.SteamManifests where sm.DepotId == downloadEvent.DownloadIdentifier orderby sm.CreationTime descending @@ -61,6 +63,7 @@ orderby downloadEvent.LastUpdatedAt descending { downloadEvent, steamDepot, + epicManifest, steamManifest }; @@ -72,8 +75,28 @@ orderby downloadEvent.LastUpdatedAt descending var mappedResult = groupedResult.Select(group => { - var firstItemInGroup = group.First(); + var firstItemInGroup = group + .FirstOrDefault(item => item.steamDepot != null) + ?? group.FirstOrDefault(item => item.epicManifest != null) + ?? group.FirstOrDefault(); + + var downloadInfo = new DownloadInfo(); + if (group.Key.CacheIdentifier == "steam") + { + downloadInfo.Name = _steamAppObtainerService.GetSteamAppById(firstItemInGroup?.steamDepot?.SteamAppId)?.name; + downloadInfo.AppUrl = $"https://steamdb.info/app/{firstItemInGroup?.steamDepot?.SteamAppId}/"; + downloadInfo.AppImageUrl = $"https://cdn.cloudflare.steamstatic.com/steam/apps/{firstItemInGroup?.steamDepot?.SteamAppId}/header.jpg"; + downloadInfo.DownloadIdentifier = group.Key.DownloadIdentifierString; + downloadInfo.DownloadIdentifierUrl = $"https://steamdb.info/depot/{group.Key.DownloadIdentifierString}"; + //downloadInfo.DownloadIdentifierImageUrl = $"https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/{group.Key.DownloadIdentifierString}/header.jpg"; + downloadInfo.TotalBytes = firstItemInGroup?.steamManifest?.TotalCompressedSize ?? 0 + firstItemInGroup?.steamManifest?.ManifestBytesSize ?? 0; + } + else if (group.Key.CacheIdentifier == "epicgames") + { + downloadInfo.Name = firstItemInGroup?.epicManifest?.Name; + downloadInfo.TotalBytes = firstItemInGroup?.epicManifest?.TotalCompressedSize ?? 0 + firstItemInGroup?.epicManifest?.ManifestBytesSize ?? 0; + } var downloadEvent = new DownloadEvent { @@ -86,21 +109,17 @@ orderby downloadEvent.LastUpdatedAt descending LastUpdatedAt = group.Key.LastUpdatedAt, CacheHitBytes = group.Key.CacheHitBytes, CacheMissBytes = group.Key.CacheMissBytes, - TotalBytes = (firstItemInGroup.steamManifest?.TotalCompressedSize ?? 0) + (firstItemInGroup.steamManifest?.ManifestBytesSize ?? 0), - SteamDepot = group.Key.CacheIdentifier != "steam" || firstItemInGroup.steamDepot == null - ? null - : new SteamDepot - { - Id = firstItemInGroup.steamDepot.SteamDepotId, - SteamAppId = firstItemInGroup.steamDepot.SteamAppId - } + //TotalBytes = (firstItemInGroup.steamManifest?.TotalCompressedSize ?? 0) + (firstItemInGroup.steamManifest?.ManifestBytesSize ?? 0), + //SteamDepot = group.Key.CacheIdentifier != "steam" || firstItemInGroup.steamDepot == null + // ? null + // : new SteamDepot + // { + // Id = firstItemInGroup.steamDepot.SteamDepotId, + // SteamAppId = firstItemInGroup.steamDepot.SteamAppId + // } + DownloadInfo = downloadInfo }; - if (downloadEvent.CacheIdentifier == "steam" && downloadEvent.SteamDepot != null) - { - downloadEvent.SteamDepot.SteamApp = _steamAppObtainerService.GetSteamAppById(downloadEvent.SteamDepot.SteamAppId); - } - return downloadEvent; }).ToList(); diff --git a/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs b/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs index 5d0cb4a..8db605b 100644 --- a/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs +++ b/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs @@ -18,9 +18,9 @@ public class DownloadEvent public long CacheHitBytes { get; set; } public long CacheMissBytes { get; set; } - public ulong TotalBytes { get; set; } + //public ulong TotalBytes { get; set; } - public SteamDepot? SteamDepot { get; set; } + public DownloadInfo? DownloadInfo { get; set; } } } diff --git a/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs b/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs new file mode 100644 index 0000000..4104f90 --- /dev/null +++ b/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs @@ -0,0 +1,17 @@ +namespace DeveLanCacheUI_Backend.Controllers.Models +{ + public class DownloadInfo + { + public string? Name { get; set; } + + public string? AppUrl { get; set; } + public string? AppImageUrl { get; set; } + + //E.g. the DepotId + public string? DownloadIdentifier { get; set; } + public string? DownloadIdentifierUrl { get; set; } + public string? DownloadIdentifierImageUrl { get; set; } + + public ulong? TotalBytes { get; set; } + } +} diff --git a/DeveLanCacheUI_Backend/Controllers/Models/SteamDepot.cs b/DeveLanCacheUI_Backend/Controllers/Models/SteamDepot.cs deleted file mode 100644 index 48ea943..0000000 --- a/DeveLanCacheUI_Backend/Controllers/Models/SteamDepot.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace DeveLanCacheUI_Backend.Controllers.Models -{ - public class SteamDepot - { - public uint Id { get; set; } - - public App? SteamApp { get; set; } - public uint? SteamAppId { get; set; } - } -} diff --git a/DeveLanCacheUI_Backend/Db/DbModels/DbAsyncLogEntryProcessingQueueItem.cs b/DeveLanCacheUI_Backend/Db/DbModels/DbAsyncLogEntryProcessingQueueItem.cs new file mode 100644 index 0000000..b151902 --- /dev/null +++ b/DeveLanCacheUI_Backend/Db/DbModels/DbAsyncLogEntryProcessingQueueItem.cs @@ -0,0 +1,10 @@ +namespace DeveLanCacheUI_Backend.Db.DbModels +{ + public class DbAsyncLogEntryProcessingQueueItem + { + [Key] + public int Id { get; set; } + + public LanCacheLogEntryRaw LanCacheLogEntryRaw { get; set; } = null!; + } +} diff --git a/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs b/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs index ced9831..a20ea83 100644 --- a/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs +++ b/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs @@ -8,6 +8,7 @@ public class DbDownloadEvent //steam/epicgames/wsus/epicgames public required string CacheIdentifier { get; set; } + //Ideally only use this for faster Joins, all other code should use the DownloadIdentifierString public uint? DownloadIdentifier { get; set; } public string? DownloadIdentifierString { get; set; } diff --git a/DeveLanCacheUI_Backend/Db/DbModels/DbEpicManifest.cs b/DeveLanCacheUI_Backend/Db/DbModels/DbEpicManifest.cs new file mode 100644 index 0000000..acdcda1 --- /dev/null +++ b/DeveLanCacheUI_Backend/Db/DbModels/DbEpicManifest.cs @@ -0,0 +1,16 @@ +namespace DeveLanCacheUI_Backend.Db.DbModels +{ + [PrimaryKey(nameof(DownloadIdentifier), nameof(CreationTime))] + public class DbEpicManifest + { + public required string DownloadIdentifier { get; set; } + public required DateTime CreationTime { get; set; } + + public required ulong TotalCompressedSize { get; set; } + public required ulong TotalUncompressedSize { get; set; } + + public required ulong ManifestBytesSize { get; set; } + + public string Name { get; set; } + } +} diff --git a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs index 4e0b3da..aa23e3d 100644 --- a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs +++ b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs @@ -8,6 +8,9 @@ public class DeveLanCacheUIDbContext : DbContext public DbSet DownloadEvents => Set(); public DbSet Settings => Set(); public DbSet SteamManifests => Set(); + public DbSet EpicManifests => Set(); + public DbSet AsyncLogEntryProcessingQueueItems => Set(); + public DeveLanCacheUIDbContext(DbContextOptions options) : base(options) { @@ -26,6 +29,37 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasKey(pc => new { pc.SteamDepotId, pc.SteamAppId }); + + // Configure the owned entity to make eligible properties optional + modelBuilder.Entity() + .OwnsOne(pc => pc.LanCacheLogEntryRaw, ownedBuilder => + { + // Define which properties should remain required regardless of type + var requiredProperties = new HashSet { "CacheIdentifier", "OriginalLogLine" }; + + // Process each property + foreach (var property in typeof(LanCacheLogEntryRaw).GetProperties()) + { + // Skip required properties + if (requiredProperties.Contains(property.Name)) + continue; + + var propertyType = property.PropertyType; + + // Check if the property type can be nullable in the database: + // 1. Reference types (string, classes, etc.) + // 2. Already nullable value types (int?, DateTime?, etc.) + bool canBeNullable = !propertyType.IsValueType || + (propertyType.IsGenericType && + propertyType.GetGenericTypeDefinition() == typeof(Nullable<>)); + + if (canBeNullable) + { + // Only configure properties that can be nullable + ownedBuilder.Property(propertyType, property.Name).IsRequired(false); + } + } + }); } protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) diff --git a/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj b/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj index 1ac4689..b7f0532 100644 --- a/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj +++ b/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj @@ -40,4 +40,8 @@ + + + + diff --git a/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs b/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs new file mode 100644 index 0000000..fbb6547 --- /dev/null +++ b/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs @@ -0,0 +1,63 @@ +using DeveLanCacheUI_Backend.Services; +using ProtoBuf.Meta; + +namespace DeveLanCacheUI_Backend.LogReading +{ + public class AsyncLogEntryProcessingQueueItemsProcessorHostedService : BackgroundService + { + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public AsyncLogEntryProcessingQueueItemsProcessorHostedService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + await GoRun(stoppingToken); + } + + private async Task GoRun(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + await using (var scope = _services.CreateAsyncScope()) + { + using var dbContext = scope.ServiceProvider.GetRequiredService(); + + // Take 1 because I think it's nicer. We could take more but then we need to do a SaveChanges inside the manifest services. + // Because if you don't you're not getting the latest data. + var items = await dbContext.AsyncLogEntryProcessingQueueItems.OrderBy(t => t.Id).Take(1).ToListAsync(stoppingToken); + if (items.Count == 0) + { + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + continue; + } + + var steamManifestService = scope.ServiceProvider.GetRequiredService(); + var epicManifestService = scope.ServiceProvider.GetRequiredService(); + + foreach (var item in items) + { + if (item.LanCacheLogEntryRaw.CacheIdentifier == "steam") + { + await steamManifestService.TryToDownloadManifest(item.LanCacheLogEntryRaw); + } + if (item.LanCacheLogEntryRaw.CacheIdentifier == "epicgames") + { + await epicManifestService.TryToDownloadManifest(item.LanCacheLogEntryRaw); + } + } + + dbContext.AsyncLogEntryProcessingQueueItems.RemoveRange(items); + await dbContext.SaveChangesAsync(stoppingToken); + } + } + } + } +} diff --git a/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs b/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs index 1c24e89..6b121c4 100644 --- a/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs +++ b/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs @@ -9,7 +9,6 @@ public class LanCacheLogReaderHostedService : BackgroundService private readonly IServiceProvider _services; private readonly DeveLanCacheConfiguration _deveLanCacheConfiguration; - private readonly SteamManifestService _steamManifestService; private readonly ILogger _logger; /// @@ -43,12 +42,10 @@ public class LanCacheLogReaderHostedService : BackgroundService public LanCacheLogReaderHostedService(IServiceProvider services, DeveLanCacheConfiguration deveLanCacheConfiguration, - SteamManifestService steamManifestService, ILogger logger) { _services = services; _deveLanCacheConfiguration = deveLanCacheConfiguration; - _steamManifestService = steamManifestService; _logger = logger; } @@ -121,15 +118,27 @@ await retryPolicy.ExecuteAsync(async () => foreach (var lanCacheLogLine in filteredLogLines) { - if (lanCacheLogLine.CacheIdentifier == "steam" && ExcludedAppIds.Contains(lanCacheLogLine.DownloadIdentifier)) + if (lanCacheLogLine.CacheIdentifier == "steam" && lanCacheLogLine.DownloadIdentifier != null && ExcludedAppIds.Contains(lanCacheLogLine.DownloadIdentifier)) { continue; } - if (lanCacheLogLine.CacheIdentifier == "steam" && lanCacheLogLine.Request.Contains("/manifest/") && DateTime.Now < lanCacheLogLine.DateTime.AddDays(14)) + + if ( + (lanCacheLogLine.CacheIdentifier == "steam" && lanCacheLogLine.Request.Contains("/manifest/")) || + (lanCacheLogLine.CacheIdentifier == "epicgames" && lanCacheLogLine.Request.Contains(".manifest?")) + ) { - _logger.LogInformation("Found manifest for Depot: {DownloadIdentifier}", lanCacheLogLine.DownloadIdentifier); - var ttt = lanCacheLogLine; - _steamManifestService.TryToDownloadManifest(ttt); + if (DateTime.Now < lanCacheLogLine.DateTime.AddDays(14)) + { + // We found a manifest. This contains more information about the download. We'll process these asynchronously + // since it requires downloading and parsing the manifest file. + _logger.LogInformation("Found manifest for download: {DownloadIdentifier}", lanCacheLogLine.DownloadIdentifier); + var itemToProcessAsynchronously = new DbAsyncLogEntryProcessingQueueItem() + { + LanCacheLogEntryRaw = lanCacheLogLine + }; + dbContext.AsyncLogEntryProcessingQueueItems.Add(itemToProcessAsynchronously); + } } var cacheKey = $"{lanCacheLogLine.CacheIdentifier}_||_{lanCacheLogLine.DownloadIdentifier}_||_{lanCacheLogLine.RemoteAddress}"; @@ -248,39 +257,39 @@ public IEnumerable> Batch2(IEnumerable TailFrom(string file, CancellationToken stoppingToken) - { - using (var reader = File.OpenText(file)) - { - // go to end - if the next line is commented out, all the lines from the beginning is returned - // reader.BaseStream.Seek(0, SeekOrigin.End); - while (true) - { - stoppingToken.ThrowIfCancellationRequested(); - - string? line = reader.ReadLine(); - if (reader.BaseStream.Length < reader.BaseStream.Position) - { - Console.WriteLine($"Uhh: {reader.BaseStream.Length} < {reader.BaseStream.Position}"); - //reader.BaseStream.Seek(0, SeekOrigin.Begin); - - } - - if (line != null) - { - yield return line; - } - else - { - yield return null; - } - } - } - } + //static IEnumerable TailFrom(string file, CancellationToken stoppingToken) + //{ + // using (var reader = File.OpenText(file)) + // { + // // go to end - if the next line is commented out, all the lines from the beginning is returned + // // reader.BaseStream.Seek(0, SeekOrigin.End); + // while (true) + // { + // stoppingToken.ThrowIfCancellationRequested(); + + // string? line = reader.ReadLine(); + // if (reader.BaseStream.Length < reader.BaseStream.Position) + // { + // Console.WriteLine($"Uhh: {reader.BaseStream.Length} < {reader.BaseStream.Position}"); + // //reader.BaseStream.Seek(0, SeekOrigin.Begin); + + // } + + // if (line != null) + // { + // yield return line; + // } + // else + // { + // yield return null; + // } + // } + // } + //} public long TotalBytesRead { get; set; } - public IEnumerable TailFrom2(Stream inputStream, CancellationToken stoppingToken) + public IEnumerable TailFrom2(Stream inputStream, CancellationToken stoppingToken) { if (inputStream.Length >= TotalBytesRead) { diff --git a/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs b/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs index ed9aa5d..87931b5 100644 --- a/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs +++ b/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs @@ -32,6 +32,7 @@ public class LanCacheLogEntryRaw public string HttpRange { get; init; } public required string OriginalLogLine { get; init; } + public string ParseException { get; init; } public DateTime DateTime { get; private set; } @@ -70,7 +71,21 @@ public void CalculateFields() } else if (CacheIdentifier == "epicgames") { - DownloadIdentifier = "unknown"; + var urlPart = Request.Split(' ')[1]; + var splitted = urlPart.Split('/', StringSplitOptions.RemoveEmptyEntries); + + if (urlPart.StartsWith("/Builds/Org/") && (splitted.Length > 2)) + { + DownloadIdentifier = splitted[2]; + } + else if (urlPart.StartsWith("/ias/")) + { + DownloadIdentifier = splitted[1]; + } + else + { + DownloadIdentifier = "unknown"; + } } else if (CacheIdentifier == "riot") { diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.Designer.cs new file mode 100644 index 0000000..5204d20 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.Designer.cs @@ -0,0 +1,207 @@ +// +using System; +using DeveLanCacheUI_Backend.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + [DbContext(typeof(DeveLanCacheUIDbContext))] + [Migration("20250506212325_AsyncLogEntryTable")] + partial class AsyncLogEntryTable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); + + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); + + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CacheIdentifier"); + + b.HasIndex("ClientIp"); + + b.ToTable("DownloadEvents"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => + { + b.Property("Key") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); + + b.Property("SteamAppId") + .HasColumnType("INTEGER"); + + b.HasKey("SteamDepotId", "SteamAppId"); + + b.ToTable("SteamDepots"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); + + b1.Property("Status") + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.cs b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.cs new file mode 100644 index 0000000..7e2535e --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.cs @@ -0,0 +1,52 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + /// + public partial class AsyncLogEntryTable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AsyncLogEntryProcessingQueueItems", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + LanCacheLogEntryRaw_CacheIdentifier = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_RemoteAddress = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_ForwardedFor = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_RemoteUser = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_TimeLocal = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Request = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Status = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_BodyBytesSent = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Referer = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_UserAgent = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_UpstreamCacheStatus = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Host = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_HttpRange = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_OriginalLogLine = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_ParseException = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_DateTime = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_BodyBytesSentLong = table.Column(type: "INTEGER", nullable: false), + LanCacheLogEntryRaw_DownloadIdentifier = table.Column(type: "TEXT", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_AsyncLogEntryProcessingQueueItems", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AsyncLogEntryProcessingQueueItems"); + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.Designer.cs new file mode 100644 index 0000000..fba60d3 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.Designer.cs @@ -0,0 +1,233 @@ +// +using System; +using DeveLanCacheUI_Backend.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + [DbContext(typeof(DeveLanCacheUIDbContext))] + [Migration("20250511104218_EpicManifestsAndRewrite")] + partial class EpicManifestsAndRewrite + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); + + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); + + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CacheIdentifier"); + + b.HasIndex("ClientIp"); + + b.ToTable("DownloadEvents"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbEpicManifest", b => + { + b.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.HasKey("DownloadIdentifier", "CreationTime"); + + b.ToTable("EpicManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => + { + b.Property("Key") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); + + b.Property("SteamAppId") + .HasColumnType("INTEGER"); + + b.HasKey("SteamDepotId", "SteamAppId"); + + b.ToTable("SteamDepots"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); + + b1.Property("Status") + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.cs b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.cs new file mode 100644 index 0000000..ff72f20 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + /// + public partial class EpicManifestsAndRewrite : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "EpicManifests", + columns: table => new + { + DownloadIdentifier = table.Column(type: "TEXT", nullable: false), + CreationTime = table.Column(type: "TEXT", nullable: false), + TotalCompressedSize = table.Column(type: "INTEGER", nullable: false), + TotalUncompressedSize = table.Column(type: "INTEGER", nullable: false), + ManifestBytesSize = table.Column(type: "INTEGER", nullable: false), + Name = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_EpicManifests", x => new { x.DownloadIdentifier, x.CreationTime }); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "EpicManifests"); + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs index eedec3e..11abef6 100644 --- a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs +++ b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs @@ -15,7 +15,18 @@ partial class DeveLanCacheUIDbContextModelSnapshot : ModelSnapshot protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.0"); + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => { @@ -58,6 +69,32 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("DownloadEvents"); }); + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbEpicManifest", b => + { + b.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.HasKey("DownloadIdentifier", "CreationTime"); + + b.ToTable("EpicManifests"); + }); + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => { b.Property("Key") @@ -112,6 +149,81 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("SteamManifests"); }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); + + b1.Property("Status") + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); #pragma warning restore 612, 618 } } diff --git a/DeveLanCacheUI_Backend/Program.cs b/DeveLanCacheUI_Backend/Program.cs index 56b23ed..0153663 100644 --- a/DeveLanCacheUI_Backend/Program.cs +++ b/DeveLanCacheUI_Backend/Program.cs @@ -67,11 +67,14 @@ public static void Main(string[] args) builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); builder.Services.AddHttpClient(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + + builder.Services.AddScoped(); + builder.Services.AddScoped(); if (deveLanCacheConfiguration.Feature_DirectSteamIntegration) { diff --git a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs new file mode 100644 index 0000000..1b02936 --- /dev/null +++ b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs @@ -0,0 +1,94 @@ + +namespace DeveLanCacheUI_Backend.Services +{ + public class EpicManifestService + { + private readonly DeveLanCacheUIDbContext _dbContext; + private readonly IHttpClientFactory _httpClientFactoryForManifestDownloads; + private readonly ILogger _logger; + private readonly string _manifestDirectory; + + public EpicManifestService( + DeveLanCacheConfiguration deveLanCacheConfiguration, + DeveLanCacheUIDbContext dbContext, + IHttpClientFactory httpClientFactory, + ILogger logger) + { + _dbContext = dbContext; + _httpClientFactoryForManifestDownloads = httpClientFactory; + _logger = logger; + + var deveLanCacheUIDataDirectory = deveLanCacheConfiguration.DeveLanCacheUIDataDirectory ?? string.Empty; + _manifestDirectory = Path.Combine(deveLanCacheUIDataDirectory, "manifests"); + } + + public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) + { + if (lanCacheLogEntryRaw.CacheIdentifier != "epicgames" || !lanCacheLogEntryRaw.Request.Contains(".manifest?")) + { + _logger.LogError("Code bug: Trying to download manifest that isn't actually a manifest: {OriginalLogLine}", lanCacheLogEntryRaw.OriginalLogLine); + return; + } + + var firstItem = await _dbContext.EpicManifests.FirstOrDefaultAsync(t => t.DownloadIdentifier == lanCacheLogEntryRaw.DownloadIdentifier); + if (firstItem != null) + { + return; + } + + + var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; + var fullManifestUrl = $"http://{lanCacheLogEntryRaw.Host}{theManifestUrlPart}"; + + var fallbackPolicy = Policy + .Handle() + .FallbackAsync(async (ct) => + { + await Task.CompletedTask; + _logger.LogInformation("Manifest saving: All retries failed, skipping..."); + }); + + var retryPolicy = Policy + .Handle() + .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + (exception, timeSpan, context) => + { + _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); + }); + + await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + { + + using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); + httpClient.DefaultRequestHeaders.Add("User-Agent", lanCacheLogEntryRaw.UserAgent); + httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again + var manifestResponse = await httpClient.GetAsync(fullManifestUrl); + + if (!manifestResponse.IsSuccessStatusCode) + { + _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); + return; + } + var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); + + var parsedManifest = EpicManifestParser.EpicManifestParser.Deserialize(manifestBytes); + + _logger.LogInformation("Parsed LogLine from: {LogLineDateTime}, Epic Manifest for AppId: '{AppId}' AppName: '{AppName}' Launch Exe: '{Manifest}'", lanCacheLogEntryRaw.DateTime, parsedManifest.Meta.AppID, parsedManifest.Meta.AppName, parsedManifest.Meta.LaunchExe); + + var epicManifest = new DbEpicManifest() + { + Name = Path.GetFileNameWithoutExtension(parsedManifest.Meta.LaunchExe), + DownloadIdentifier = lanCacheLogEntryRaw.DownloadIdentifier, + CreationTime = DateTime.UtcNow, + ManifestBytesSize = (ulong)manifestBytes.Length, + TotalCompressedSize = (ulong)parsedManifest.TotalDownloadSize, + TotalUncompressedSize = (ulong)parsedManifest.TotalBuildSize + }; + + await _dbContext.EpicManifests.AddAsync(epicManifest); + await _dbContext.SaveChangesAsync(); + }); + } + } +} diff --git a/DeveLanCacheUI_Backend/Services/SteamManifestService.cs b/DeveLanCacheUI_Backend/Services/SteamManifestService.cs index e5a1522..eeb3372 100644 --- a/DeveLanCacheUI_Backend/Services/SteamManifestService.cs +++ b/DeveLanCacheUI_Backend/Services/SteamManifestService.cs @@ -2,16 +2,18 @@ { public class SteamManifestService { - private readonly IServiceProvider _services; + private readonly DeveLanCacheUIDbContext _dbContext; private readonly IHttpClientFactory _httpClientFactoryForManifestDownloads; private readonly ILogger _logger; private readonly string _manifestDirectory; - private static SemaphoreSlim _semaphoreSlim = new SemaphoreSlim(1, 1); - - public SteamManifestService(DeveLanCacheConfiguration deveLanCacheConfiguration, IServiceProvider services, IHttpClientFactory httpClientFactory, ILogger logger) + public SteamManifestService( + DeveLanCacheConfiguration deveLanCacheConfiguration, + DeveLanCacheUIDbContext dbContext, + IHttpClientFactory httpClientFactory, + ILogger logger) { - _services = services; + _dbContext = dbContext; _httpClientFactoryForManifestDownloads = httpClientFactory; _logger = logger; @@ -19,112 +21,91 @@ public SteamManifestService(DeveLanCacheConfiguration deveLanCacheConfiguration, _manifestDirectory = Path.Combine(deveLanCacheUIDataDirectory, "manifests"); } - public void TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) + public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) { - //This method could use some TPL Dataflow, I now use locking which should be okayish - - if (!lanCacheLogEntryRaw.Request.Contains("/manifest/") || lanCacheLogEntryRaw.DownloadIdentifier == null) + if (lanCacheLogEntryRaw.CacheIdentifier != "steam" || !lanCacheLogEntryRaw.Request.Contains("/manifest/")) { _logger.LogError("Code bug: Trying to download manifest that isn't actually a manifest: {OriginalLogLine}", lanCacheLogEntryRaw.OriginalLogLine); return; } - _ = Task.Run(async () => - { - var fallbackPolicy = Policy - .Handle() - .FallbackAsync(async (ct) => - { - await Task.CompletedTask; - _logger.LogInformation("Manifest saving: All retries failed, skipping..."); - }); - - var retryPolicy = Policy - .Handle() - .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), - (exception, timeSpan, context) => - { - _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); - }); - - await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + var fallbackPolicy = Policy + .Handle() + .FallbackAsync(async (ct) => { - try - { - _semaphoreSlim.Wait(); - await using (var scope = _services.CreateAsyncScope()) - { - var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; - - var everythingAfterManifest = theManifestUrlPart.Split("/manifest/", StringSplitOptions.RemoveEmptyEntries).Last(); - var manifestId = everythingAfterManifest.Split("/", StringSplitOptions.RemoveEmptyEntries).First(); - - //Replace invalid chars should dissalow reading any file you want :) - var manifestIdFileName = RemoveNonNumericCharacters(manifestId) + ".bin"; - var depotId = RemoveNonNumericCharacters(lanCacheLogEntryRaw.DownloadIdentifier!); - var depotIdAndManifestIdentifier = Path.Combine(depotId, manifestIdFileName); + await Task.CompletedTask; + _logger.LogInformation("Manifest saving: All retries failed, skipping..."); + }); + var retryPolicy = Policy + .Handle() + .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + (exception, timeSpan, context) => + { + _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); + }); - using var dbContext = scope.ServiceProvider.GetRequiredService(); - var dbManifestFound = await dbContext.SteamManifests.FirstOrDefaultAsync(t => t.UniqueManifestIdentifier == depotIdAndManifestIdentifier); + await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + { + var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; - if (dbManifestFound != null) - { - return; - } + var everythingAfterManifest = theManifestUrlPart.Split("/manifest/", StringSplitOptions.RemoveEmptyEntries).Last(); + var manifestId = everythingAfterManifest.Split("/", StringSplitOptions.RemoveEmptyEntries).First(); - var fullPath = Path.Combine(_manifestDirectory, depotIdAndManifestIdentifier); - Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + //Replace invalid chars should dissalow reading any file you want :) + var manifestIdFileName = RemoveNonNumericCharacters(manifestId) + ".bin"; + var depotId = RemoveNonNumericCharacters(lanCacheLogEntryRaw.DownloadIdentifier!); + var depotIdAndManifestIdentifier = Path.Combine(depotId, manifestIdFileName); + var dbManifestFound = await _dbContext.SteamManifests.FirstOrDefaultAsync(t => t.UniqueManifestIdentifier == depotIdAndManifestIdentifier); - var cachedUrl = $"http://lancache.steamcontent.com{theManifestUrlPart}"; - using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); - httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); - httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); - httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again - var manifestResponse = await httpClient.GetAsync(cachedUrl); + if (dbManifestFound != null) + { + return; + } + var fullPath = Path.Combine(_manifestDirectory, depotIdAndManifestIdentifier); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + var cachedUrl = $"http://lancache.steamcontent.com{theManifestUrlPart}"; + using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); + httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); + httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again + var manifestResponse = await httpClient.GetAsync(cachedUrl); + if (!manifestResponse.IsSuccessStatusCode) + { + _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); + return; + } + var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); - if (!manifestResponse.IsSuccessStatusCode) - { - _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); - return; - } - var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); + var dbManifest = ManifestBytesToDbSteamManifest(manifestBytes, depotIdAndManifestIdentifier); - var dbManifest = ManifestBytesToDbSteamManifest(manifestBytes, depotIdAndManifestIdentifier); + if (dbManifest == null) + { + _logger.LogWarning("Could not get manifest for depot: {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + return; + } - if (dbManifest == null) - { - _logger.LogWarning("Could not get manifest for depot: {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - return; - } + var dbValue = _dbContext.SteamManifests.FirstOrDefault(t => t.DepotId == dbManifest.DepotId && t.CreationTime == dbManifest.CreationTime); + if (dbValue != null) + { + _dbContext.Entry(dbValue).CurrentValues.SetValues(dbManifest); + _logger.LogInformation("Updated manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + } + else + { + await _dbContext.SteamManifests.AddAsync(dbManifest); + _logger.LogInformation("Added manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + } - var dbValue = dbContext.SteamManifests.FirstOrDefault(t => t.DepotId == dbManifest.DepotId && t.CreationTime == dbManifest.CreationTime); - if (dbValue != null) - { - dbContext.Entry(dbValue).CurrentValues.SetValues(dbManifest); - _logger.LogInformation("Updated manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - } - else - { - await dbContext.SteamManifests.AddAsync(dbManifest); - _logger.LogInformation("Added manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - } + await File.WriteAllBytesAsync(fullPath, manifestBytes); - await File.WriteAllBytesAsync(fullPath, manifestBytes); - await dbContext.SaveChangesAsync(); - } - } - finally - { - _semaphoreSlim.Release(); - } - }); + // Save changes happens in the caller }); } diff --git a/DeveLanCacheUI_Backend/appsettings.Development.json b/DeveLanCacheUI_Backend/appsettings.Development.json index 8739a29..f54bbca 100644 --- a/DeveLanCacheUI_Backend/appsettings.Development.json +++ b/DeveLanCacheUI_Backend/appsettings.Development.json @@ -1,11 +1,11 @@ { "Logging": { "LogLevel": { - "DeveLanCacheUI_Backend.Services": "Trace" + //"DeveLanCacheUI_Backend.Services": "Trace" } }, "DeveLanCacheUIDataDirectory": "DeveLanCacheUIData", - "LanCacheLogsDirectory": "\\\\10.88.20.1\\DockerComposers\\lancache\\logs", + "LanCacheLogsDirectory": "\\\\10.88.43.1\\root\\dockercomposers\\lancache\\logs", "Feature_DirectSteamIntegration": true, "Feature_SkipLinesBasedOnBytesRead": true, "ExcludedClientIps": ""