reorg for readability in program code and in UI

+ Reorganize namespaces for readability in program code.
+ Change descriptions of file trees in UI for more consistency.

Author: Viir

implement/Pine.Core/Files/CommonMappings.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pine.Core.Files;
+
+/// <summary>
+/// Provides utility methods for extracting tree structures from named binary blobs, supporting common archive formats
+/// such as ZIP, TAR.GZ, TGZ, and GZ.
+/// </summary>
+/// <remarks>Use the methods in this class to interpret and extract hierarchical data from blobs based on their
+/// file name and format. The extraction logic supports several widely used archive types, enabling consistent access to
+/// tree representations regardless of the underlying compression or packaging. This class is intended for scenarios
+/// where blob content may represent an archive or compressed file and a tree structure is required for further
+/// processing.</remarks>
+public class CommonMappings
+{
+    /// <summary>
+    /// Extracts one or more tree structures from the specified named blob, interpreting its content as a supported
+    /// archive or compressed format.
+    /// </summary>
+    /// <remarks>The method attempts to extract trees from the blob by interpreting its content according to
+    /// the file extension. Supported formats include ZIP, TAR.GZ, TGZ, and GZ. If the content cannot be parsed as a
+    /// supported archive, no trees are returned. The method does not throw exceptions for invalid or unsupported
+    /// formats; instead, it silently skips extraction for those cases.</remarks>
+    /// <param name="blobName">The name of the blob, including its file extension, which determines how the content is interpreted (e.g.,
+    /// ".zip", ".tar.gz", ".tgz", ".gz").</param>
+    /// <param name="blobContent">The binary content of the blob to be processed. Must represent a valid archive or compressed file format
+    /// supported by the method.</param>
+    /// <returns>An enumerable collection of tree structures extracted from the blob content. The collection may contain zero or
+    /// more trees, depending on the format and validity of the content.</returns>
+
+    public static IEnumerable<BlobTreeWithStringPath> ExtractTreesFromNamedBlob(
+        string blobName,
+        ReadOnlyMemory<byte> blobContent)
+    {
+        {
+            BlobTreeWithStringPath? fromZipArchive = null;
+
+            try
+            {
+                fromZipArchive =
+                    PineValueComposition.SortedTreeFromSetOfBlobsWithCommonFilePath(
+                        ZipArchive.EntriesFromZipArchive(blobContent));
+            }
+            catch { }
+
+            if (fromZipArchive is not null)
+                yield return fromZipArchive;
+        }
+
+        if (blobName.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase) ||
+            blobName.EndsWith(".tgz", StringComparison.OrdinalIgnoreCase))
+        {
+            BlobTreeWithStringPath? fromTarArchive = null;
+
+            try
+            {
+                fromTarArchive =
+                    TarArchive.TreeWithStringPathFromTarArchive(BytesConversions.DecompressGzip(blobContent));
+            }
+            catch { }
+
+            if (fromTarArchive is not null)
+                yield return fromTarArchive;
+        }
+        else
+        {
+            if (blobName.EndsWith(".gz", StringComparison.OrdinalIgnoreCase))
+            {
+                ReadOnlyMemory<byte>? fromGzip = null;
+
+                try
+                {
+                    fromGzip = BytesConversions.DecompressGzip(blobContent);
+                }
+                catch { }
+
+                if (fromGzip is not null)
+                    yield return BlobTreeWithStringPath.Blob(fromGzip.Value);
+            }
+        }
+    }
+}

implement/Pine.Core/Files/FileTreeExtensions.cs

@@ -0,0 +1,69 @@
+using Pine.Core.Addressing;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
+namespace Pine.Core.Files;
+
+/// <summary>
+/// Extension methods for <see cref="BlobTreeWithStringPath"/>.
+/// </summary>
+public static class FileTreeExtensions
+{
+    /// <summary>
+    /// Generates human-readable overviews of a file tree or blob, for use in command-line interfaces or logs.
+    /// </summary>
+    public static IEnumerable<string> DescribeFileTreeForHumans(
+        BlobTreeWithStringPath composition,
+        bool listFiles,
+        string? extractFileName)
+    {
+        if (composition is BlobTreeWithStringPath.TreeNode tree)
+        {
+            var blobs = composition.EnumerateBlobsTransitive().ToImmutableList();
+
+            yield return
+                "a directory containing " + blobs.Count + " files with an aggregate size of " +
+                CommandLineInterface.FormatIntegerForDisplay(blobs.Sum(blob => (long)blob.blobContent.Length)) + " bytes.";
+
+            if (listFiles)
+            {
+                yield return
+                    "file paths, sizes and hashes:\n" +
+                    string.Join(
+                        "\n",
+                        blobs.Select(blobAtPath =>
+                        string.Join("/", blobAtPath.path) + " : " +
+                        blobAtPath.blobContent.Length + " bytes, " +
+                        Convert.ToHexStringLower(PineValueHashTree.ComputeHash(PineValue.Blob(blobAtPath.blobContent)).Span)[..10]));
+            }
+
+            yield break;
+        }
+
+        if (composition is BlobTreeWithStringPath.BlobNode blob)
+        {
+            yield return "a blob containing " + blob.Bytes.Length + " bytes";
+
+            if (extractFileName is not null)
+            {
+                foreach (var extractedTree in CommonMappings.ExtractTreesFromNamedBlob(extractFileName, blob.Bytes))
+                {
+                    var extractedTreeCompositionId =
+                        Convert.ToHexStringLower(PineValueHashTree.ComputeHashNotSorted(extractedTree).Span);
+
+                    var compositionDescription =
+                        string.Join(
+                            "\n",
+                            DescribeFileTreeForHumans(
+                                extractedTree,
+                                listFiles: listFiles,
+                                extractFileName: null));
+
+                    yield return "Extracted composition " + extractedTreeCompositionId + ", which is " + compositionDescription;
+                }
+            }
+        }
+    }
+}

implement/Pine.Core/Files/TarArchive.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+
+namespace Pine.Core.Files;
+
+/// <summary>
+/// Helpers for working with TAR archives.
+/// </summary>
+public static class TarArchive
+{
+    /// <summary>
+    /// Creates a tree representation of the contents of a TAR archive, using string-based paths for each entry.
+    /// </summary>
+    /// <remarks>The method reads the entire TAR archive from memory and constructs a tree where each file and
+    /// directory is accessible by its string path. The input buffer is not modified. This method is suitable for
+    /// scenarios where the archive is already loaded into memory.</remarks>
+    /// <param name="tarArchive">A read-only memory buffer containing the bytes of the TAR archive to be parsed. Must contain a valid TAR archive
+    /// format.</param>
+    public static BlobTreeWithStringPath TreeWithStringPathFromTarArchive(ReadOnlyMemory<byte> tarArchive)
+    {
+        using var archiveReader =
+            SharpCompress.Archives.Tar.TarArchive.Open(new MemoryStream(tarArchive.ToArray()));
+
+        return TreeWithStringPathFromTarArchiveEntries(archiveReader.Entries);
+    }
+
+    /// <summary>
+    /// Creates a tree structure of blobs from the file entries in a TAR archive, using string paths as keys.
+    /// </summary>
+    /// <remarks>The resulting tree preserves the original file paths from the TAR archive as string keys.
+    /// Directory entries in the archive are excluded from the tree.</remarks>
+    /// <param name="entries">A collection of TAR archive entries to include in the tree. Only file entries are processed; directory entries
+    /// are ignored.</param>
+    public static BlobTreeWithStringPath TreeWithStringPathFromTarArchiveEntries(
+        IEnumerable<SharpCompress.Archives.Tar.TarArchiveEntry> entries)
+    {
+        var treeEntries =
+            entries
+            .Where(tarEntry => !tarEntry.IsDirectory)
+            .Select(tarEntry =>
+            {
+                using var memoryStream = new MemoryStream();
+                using var tarEntryStream = tarEntry.OpenEntryStream();
+
+                tarEntryStream.CopyTo(memoryStream);
+
+                var componentBytes = memoryStream.ToArray();
+
+                return (name: tarEntry.Key, component: BlobTreeWithStringPath.Blob(componentBytes));
+            }).ToImmutableList();
+
+        return BlobTreeWithStringPath.SortedTree(treeEntries);
+    }
+}

implement/Pine.Core/Files/ZipArchive.cs

@@ -0,0 +1,165 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Pine.Core.Files;
+
+/// <summary>
+/// Helpers for working with ZIP archives.
+/// <see href="https://en.wikipedia.org/wiki/ZIP_(file_format)"></see>
+/// </summary>
+public static class ZipArchive
+{
+    /// <summary>
+    /// https://github.com/dotnet/corefx/blob/a10890f4ffe0fadf090c922578ba0e606ebdd16c/src/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs#L206-L234
+    /// </summary>
+    public static DateTimeOffset EntryLastWriteTimeDefault => new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero);
+
+    /// <summary>
+    /// Creates a ZIP archive containing the specified file entries and returns its binary data as a byte array.
+    /// </summary>
+    /// <remarks>Each file entry is added to the archive using its provided name. The method does not preserve
+    /// file metadata such as timestamps or permissions. The returned byte array can be saved to disk or transmitted as
+    /// needed.</remarks>
+    /// <param name="entries">A collection of file entries, each consisting of a file name and its content as a read-only memory buffer. Each
+    /// entry will be added to the archive with the provided name and content.</param>
+    /// <param name="compressionLevel">The compression level to apply to the archive. Defaults to <see
+    /// cref="System.IO.Compression.CompressionLevel.Optimal"/> if not specified.</param>
+    /// <returns>A byte array containing the ZIP archive data with all specified entries included.</returns>
+    public static byte[] ZipArchiveFromFiles(
+        IEnumerable<(string name, ReadOnlyMemory<byte> content)> entries,
+        System.IO.Compression.CompressionLevel compressionLevel = System.IO.Compression.CompressionLevel.Optimal) =>
+        ZipArchiveFromFiles(
+            [.. entries.Select(entry => (entry.name, entry.content, EntryLastWriteTimeDefault))],
+            compressionLevel);
+
+    /// <summary>
+    /// Creates a ZIP archive containing the specified files and returns its contents as a byte array.
+    /// </summary>
+    /// <remarks>Each file path is constructed by joining the path segments in the key with a forward slash
+    /// ('/'). The method does not validate file names or contents; callers should ensure that paths and data are valid
+    /// for their intended use.</remarks>
+    /// <param name="entries">A dictionary mapping each file's path segments to its content. Each key represents the file path as a list of
+    /// strings, and each value contains the file's data as a read-only memory buffer.</param>
+    /// <param name="compressionLevel">The compression level to use when creating the ZIP archive. The default is CompressionLevel.Optimal.</param>
+    /// <returns>A byte array containing the ZIP archive with all specified files. The array will be empty if no entries are
+    /// provided.</returns>
+    public static byte[] ZipArchiveFromFiles(
+        IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> entries,
+        System.IO.Compression.CompressionLevel compressionLevel = System.IO.Compression.CompressionLevel.Optimal) =>
+        ZipArchiveFromFiles(
+            entries.Select(entry => (name: string.Join("/", entry.Key), content: entry.Value)),
+            compressionLevel);
+
+    /// <summary>
+    /// Creates a ZIP archive containing the specified files and returns its contents as a byte array.
+    /// </summary>
+    /// <remarks>The returned ZIP archive is created in memory and is not written to disk. The method does not
+    /// validate the uniqueness of file names; duplicate names may result in multiple entries with the same name in the
+    /// archive. The last write time for each entry is set according to the provided value.</remarks>
+    /// <param name="entries">A collection of file entries to include in the archive. Each entry specifies the file name, content as a
+    /// read-only memory buffer, and the last write time to set for the file in the archive. The file name must be a
+    /// valid ZIP entry name and cannot be null or empty.</param>
+    /// <param name="compressionLevel">The compression level to use for each file in the archive. Defaults to <see
+    /// cref="System.IO.Compression.CompressionLevel.Optimal"/> if not specified.</param>
+    /// <returns>A byte array containing the complete ZIP archive. The array will be empty if no entries are provided.</returns>
+    public static byte[] ZipArchiveFromFiles(
+        IEnumerable<(string name, ReadOnlyMemory<byte> content, DateTimeOffset lastWriteTime)> entries,
+        System.IO.Compression.CompressionLevel compressionLevel = System.IO.Compression.CompressionLevel.Optimal)
+    {
+        var stream = new MemoryStream();
+
+        using (var fclZipArchive = new System.IO.Compression.ZipArchive(stream, System.IO.Compression.ZipArchiveMode.Create, true))
+        {
+            foreach (var (entryName, entryContent, lastWriteTime) in entries)
+            {
+                var entry = fclZipArchive.CreateEntry(entryName, compressionLevel);
+
+                entry.LastWriteTime = lastWriteTime;
+
+                using var entryStream = entry.Open();
+
+                entryStream.Write(entryContent.Span);
+            }
+        }
+
+        stream.Seek(0, SeekOrigin.Begin);
+
+        var zipArchive = new byte[stream.Length];
+
+        stream.Read(zipArchive, 0, (int)stream.Length);
+        stream.Dispose();
+
+        return zipArchive;
+    }
+
+    /// <summary>
+    /// Enumerates the files contained in a ZIP archive, each represented by its (flat) path and content.
+    /// </summary>
+    /// <param name="zipArchive">A read-only memory buffer containing the ZIP archive data. Must be a valid ZIP file; otherwise, the behavior is
+    /// undefined.</param>
+    /// <returns>An enumerable collection of tuples, each containing the file name and its content as a read-only memory buffer.
+    /// Only file entries are included; directory entries are excluded.</returns>
+    public static IEnumerable<(string name, ReadOnlyMemory<byte> content)> FileEntriesFromZipArchive(
+        ReadOnlyMemory<byte> zipArchive) =>
+        EntriesFromZipArchive(
+            zipArchive: zipArchive,
+            includeEntry: entry => !entry.FullName.Replace('\\', '/').EndsWith('/'));
+
+    /// <summary>
+    /// Extracts all entries from the specified ZIP archive and returns their names and contents.
+    /// </summary>
+    /// <remarks>The method returns all entries in the archive, including files and directories.
+    /// The order of entries matches their order in the archive.</remarks>
+    /// <param name="zipArchive">A read-only memory buffer containing the ZIP archive data. Must represent a valid ZIP file format.</param>
+    /// <returns>An enumerable collection of tuples, each containing the entry name and its content as a read-only memory buffer.
+    /// The collection is empty if the archive contains no entries.</returns>
+    public static IEnumerable<(string name, ReadOnlyMemory<byte> content)> EntriesFromZipArchive(ReadOnlyMemory<byte> zipArchive) =>
+        EntriesFromZipArchive(zipArchive: zipArchive, includeEntry: _ => true);
+
+    /// <summary>
+    /// Enumerates entries from a ZIP archive and returns the name and content of each entry that matches the specified
+    /// filter.
+    /// </summary>
+    /// <remarks>The method reads the entire content of each included entry into memory. Use caution when
+    /// processing large archives or entries to avoid excessive memory usage.</remarks>
+    /// <param name="zipArchive">A read-only memory buffer containing the binary data of the ZIP archive to be read.</param>
+    /// <param name="includeEntry">A predicate used to determine whether a given ZIP archive entry should be included in the results. The function
+    /// receives each entry and should return <see langword="true"/> to include the entry; otherwise, <see
+    /// langword="false"/>.</param>
+    /// <returns>An enumerable collection of tuples, each containing the entry name and its content as a read-only memory buffer.
+    /// Only entries for which <paramref name="includeEntry"/> returns <see langword="true"/> are included.</returns>
+    /// <exception cref="Exception">Thrown if the number of bytes read from an entry does not match the expected entry length.</exception>
+    public static IEnumerable<(string name, ReadOnlyMemory<byte> content)> EntriesFromZipArchive(
+        ReadOnlyMemory<byte> zipArchive,
+        Func<System.IO.Compression.ZipArchiveEntry, bool> includeEntry)
+    {
+        using var fclZipArchive =
+            new System.IO.Compression.ZipArchive(
+                new MemoryStream(zipArchive.ToArray()),
+                System.IO.Compression.ZipArchiveMode.Read);
+
+        foreach (var entry in fclZipArchive.Entries)
+        {
+            if (!includeEntry(entry))
+                continue;
+
+            using var entryStream = entry.Open();
+            using var memoryStream = new MemoryStream();
+
+            entryStream.CopyTo(memoryStream);
+
+            var entryContent = memoryStream.ToArray();
+
+            if (entryContent.Length != entry.Length)
+            {
+                throw new Exception(
+                    "Error trying to read entry '" + entry.FullName + "': got " +
+                    entryContent.Length + " bytes from entry instead of " + entry.Length);
+            }
+
+            yield return (entry.FullName, entryContent);
+        }
+    }
+}

implement/Pine.Core/Pine.Core.csproj

@@ -44,6 +44,7 @@
     <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.3.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.14.0" />
     <PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="9.0.10" />
+    <PackageReference Include="SharpCompress" Version="0.41.0" />
   </ItemGroup>
 
 </Project>

implement/Pine.IntegrationTests/LoadFromGitHubTests.cs

@@ -1,5 +1,6 @@
 using AwesomeAssertions;
 using Pine.Core;
+using Pine.Core.Files;
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;