Merge branch 'Limine-Bootloader:trunk' into trunk

Author: iczelia

bootstrap

@@ -101,7 +101,7 @@ if ! test -f version; then
     clone_repo_commit \
         https://github.com/Mintsuki/Flanterm.git \
         flanterm \
-        112434532822dde252c84b25a6798a9c00515d66
+        f3221ad399f08437efd6de77a5f0d9a5607a8649
 
     download_by_hash \
         https://github.com/nothings/stb/raw/5c205738c191bcb0abc65c4febfa9bd25ff35234/stb_image.h \

common/compress/gzip.c

@@ -56,9 +56,9 @@ static size_t gz_source_read(void * user, void * buf, size_t len) {
   uint64_t avail = gh->source->size - gh->src_pos;
   if ((uint64_t)len > avail) len = (size_t)avail;
   if (len == 0) return 0;
-  fread(gh->source, buf, gh->src_pos, len);
-  gh->src_pos += len;
-  return len;
+  size_t got = fread(gh->source, buf, gh->src_pos, len);
+  gh->src_pos += got;
+  return got;
 }
 
 /*  (Re)initialize the decoder for a fresh pass over the compressed stream.

common/compress/gzip.h

@@ -37,13 +37,10 @@ bool gzip_check(struct file_handle * fd);
  * decompresses the data.  The returned handle takes ownership of
  * `compressed` and will close it when itself is closed.
  *
- * WARNING: Due to a Gzip format deficiency, ->size of the resulting
- * file_handle is only an approximation (i.e., it is not correct for
- * files larger than 4 GiB and doesn't necessarily have to reflect
- * the genuine decompressed size at all in adversarial circumstances).
- * 
- * The real decompressed size can only be authoritatively obtained by
- * fully decompressing the file.
+ * NOTE: ->size on the returned handle is set to UINT64_MAX as a
+ * sentinel for "unknown". The gzip ISIZE trailer is decompressed-size
+ * mod 2^32 and is not parsed here. Callers must drive ->read until
+ * it returns 0 (end-of-stream) to discover the true size.
  *
  * Supports very fast sequential reads and random-access reads (with
  * an implicit rewind + skip penalty inherent to the gzip format).

common/fs/file.s2.c

@@ -83,6 +83,9 @@ void fclose(struct file_handle *fd) {
 
 uint64_t fread(struct file_handle *fd, void *buf, uint64_t loc, uint64_t count) {
     if (fd->is_memfile) {
+        if (fd->is_high_mem) {
+            panic(false, "fread: memfile resides above 4 GiB; caller must use load_addr_64 directly");
+        }
         if (loc >= fd->size || count > fd->size - loc) {
             panic(false, "fread: attempted out of bounds read");
         }

common/fs/iso9660.s2.c

@@ -143,8 +143,10 @@ static void iso9660_cache_root(struct volume *vol,
 
     *root_size = pv.root.extent_size.little;
 
-    // Validate root directory size to prevent memory exhaustion
-    if (*root_size == 0 || *root_size > ISO9660_MAX_DIR_SIZE) {
+    // Validate root directory size to prevent memory exhaustion, and require
+    // sector alignment so directory-traversal sector-skip arithmetic is sound.
+    if (*root_size == 0 || *root_size > ISO9660_MAX_DIR_SIZE
+     || *root_size % ISO9660_SECTOR_SIZE != 0) {
         panic(false, "ISO9660: Invalid root directory size");
     }
 
@@ -284,6 +286,11 @@ static struct iso9660_directory_entry *iso9660_next_entry(void *current, void *b
     if (entry->length < sizeof(struct iso9660_directory_entry))
         return NULL;
 
+    // Validate that the entire entry (as declared by its length field) is
+    // within the buffer, so callers can safely read all entry->length bytes.
+    if ((size_t)entry->length > (size_t)((uint8_t *)buffer_end - (uint8_t *)entry))
+        return NULL;
+
     return entry;
 }
 
@@ -485,8 +492,11 @@ struct file_handle *iso9660_open(struct volume *vol, const char *path) {
             pmm_free(current, current_size);
         }
 
-        // Validate directory size to prevent memory exhaustion
-        if (next_size == 0 || next_size > ISO9660_MAX_DIR_SIZE) {
+        // Validate directory size to prevent memory exhaustion, and require
+        // sector alignment so directory-traversal sector-skip arithmetic is
+        // sound.
+        if (next_size == 0 || next_size > ISO9660_MAX_DIR_SIZE
+         || next_size % ISO9660_SECTOR_SIZE != 0) {
             pmm_free(ret, sizeof(struct iso9660_file_handle));
             return NULL;
         }

common/lib/uri.c

@@ -397,8 +397,41 @@ struct file_handle *uri_open(char *uri, uint32_t type, bool allow_high_mem
 
         for (;;) {
             if (buf_len == buf_cap) {
-                // Grow: new capacity = 2x (capped to prevent absurd jumps).
-                uint64_t new_cap = buf_cap * 2;
+                // Grow: double up to 64 MiB, then add 64 MiB per step.
+                // Doubling past that wastes too much memory on large files.
+                uint64_t new_cap = buf_cap < 0x4000000
+                    ? buf_cap * 2
+                    : buf_cap + 0x4000000;
+                uint64_t delta = new_cap - buf_cap;
+
+                // Try to extend in place by claiming the USABLE range
+                // immediately below the current buffer. The allocator is
+                // top-down, so above is already taken; below is the only
+                // direction that can be contiguous. On success we only
+                // pay delta extra bytes, not 2x peak.
+                if (buf_addr >= delta &&
+                    memmap_alloc_range(buf_addr - delta, delta, type,
+                                       MEMMAP_USABLE, false, false, false)) {
+                    uint64_t base = buf_addr - delta;
+                    // Move existing data down. dest < src, forward-safe.
+#if defined (__i386__)
+                    if (is_high) {
+                        for (uint64_t off = 0; off < buf_len; off += 0x100000) {
+                            size_t chunk = buf_len - off < 0x100000 ? (size_t)(buf_len - off) : 0x100000;
+                            memcpy_from_64(pool, buf_addr + off, chunk);
+                            memcpy_to_64(base + off, pool, chunk);
+                        }
+                    } else
+#endif
+                    {
+                        memmove((void *)(uintptr_t)base, buf_low, buf_len);
+                        buf_low = (void *)(uintptr_t)base;
+                    }
+                    buf_addr = base;
+                    buf_cap = new_cap;
+                    goto grew;
+                }
+
                 void *new_low = NULL;
                 uint64_t new_addr = 0;
                 uri_alloc(new_cap, type, allow_high_mem, &new_low, &new_addr);
@@ -444,6 +477,7 @@ struct file_handle *uri_open(char *uri, uint32_t type, bool allow_high_mem
                 }
                 is_high = new_is_high;
 #endif
+grew:;
             }
 
             uint64_t want = buf_cap - buf_len;
@@ -463,6 +497,16 @@ struct file_handle *uri_open(char *uri, uint32_t type, bool allow_high_mem
             buf_len += got;
         }
 
+        // Release the page-aligned tail past the actual data so we don't
+        // hand the OS up to 64 MiB of slack typed as `type`. Keep at least
+        // one page so the returned handle has a valid address.
+        uint64_t kept = ALIGN_UP(buf_len, 4096, panic(true, "uri: alignment overflow"));
+        if (kept == 0) kept = 4096;
+        if (kept < buf_cap) {
+            uri_release_range(buf_addr + kept, buf_cap - kept);
+            buf_cap = kept;
+        }
+
 #if defined (__i386__)
         if (pool != NULL) pmm_free(pool, 0x100000);
 #endif

common/protos/limine.c

@@ -493,6 +493,10 @@ noreturn void limine_load(char *config, char *cmdline) {
     strcpy(k_path_copy, kernel_path);
     char *k_resource = NULL, *k_root = NULL, *k_path = NULL, *k_hash = NULL;
     uri_resolve(k_path_copy, &k_resource, &k_root, &k_path, &k_hash);
+    // Strip the gzip `$` marker so reuse for module paths doesn't double-prefix.
+    if (k_resource[0] == '$') {
+        k_resource++;
+    }
     // Copy k_resource and k_root since uri_resolve returns pointers to a static
     // buffer that gets overwritten by subsequent uri_open/uri_resolve calls
     k_resource = strdup(k_resource);

common/sys/lapic.c

@@ -64,8 +64,9 @@ void lapic_prep_lint(struct madt *madt, uint32_t acpi_uid, bool x2apic) {
 
                 struct madt_lapic_nmi *nmi = (void *)madt_ptr;
 
-                // Match all processors (0xff) or specific UID
-                if (nmi->acpi_processor_uid != 0xff && nmi->acpi_processor_uid != (uint8_t)acpi_uid) {
+                // Match all processors (0xff) or specific UID.
+                if (nmi->acpi_processor_uid != 0xff
+                 && (acpi_uid > 0xfe || nmi->acpi_processor_uid != acpi_uid)) {
                     continue;
                 }
 

stage1/decompressor.asm

@@ -75,6 +75,8 @@ _start:
     mov    esi, ecx
     lea    ecx, [eax+0x4]            ; count = matchlen + 4
     rep    movsb                     ; copy match
+    cmp    edx, ebx                  ; guard against streams that end on a match
+    jae    .Lcrc
     jmp    .Ltoken
     ; CRC32 verification
 .Lcrc:

tools/limlzpack.c

@@ -28,35 +28,86 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdio.h>
+#include <stdbool.h>
 
 typedef unsigned char byte;
 
+static uint16_t endswap16(uint16_t value) {
+    uint16_t ret = 0;
+    ret |= (value >> 8) & 0x00ff;
+    ret |= (value << 8) & 0xff00;
+    return ret;
+}
+
+static uint32_t endswap32(uint32_t value) {
+    uint32_t ret = 0;
+    ret |= (value >> 24) & 0x000000ff;
+    ret |= (value >> 8)  & 0x0000ff00;
+    ret |= (value << 8)  & 0x00ff0000;
+    ret |= (value << 24) & 0xff000000;
+    return ret;
+}
+
+static uint64_t endswap64(uint64_t value) {
+    uint64_t ret = 0;
+    ret |= (value >> 56) & 0x00000000000000ff;
+    ret |= (value >> 40) & 0x000000000000ff00;
+    ret |= (value >> 24) & 0x0000000000ff0000;
+    ret |= (value >> 8)  & 0x00000000ff000000;
+    ret |= (value << 8)  & 0x000000ff00000000;
+    ret |= (value << 24) & 0x0000ff0000000000;
+    ret |= (value << 40) & 0x00ff000000000000;
+    ret |= (value << 56) & 0xff00000000000000;
+    return ret;
+}
+
+#ifdef __BYTE_ORDER__
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#define bigendian true
+#else
+#define bigendian false
+#endif
+
+#else /* !__BYTE_ORDER__ */
+
+static bool bigendian = false;
+
+#endif /* !__BYTE_ORDER__ */
+
+#define ENDSWAP(VALUE) (bigendian ? (                    \
+    sizeof(VALUE) == 1 ? (VALUE)          :              \
+    sizeof(VALUE) == 2 ? endswap16(VALUE) :              \
+    sizeof(VALUE) == 4 ? endswap32(VALUE) :              \
+    sizeof(VALUE) == 8 ? endswap64(VALUE) : (abort(), 1) \
+) : (VALUE))
+
 /*  Higher -> better compression with exponentally dimnishing gains.  */
 #define LIMLZ_SA_NEIGHBORS 32
 
-struct sa_cmp_ctx { int * rank; size_t n, k; };
+struct sa_cmp_ctx { int32_t *rank; size_t n, k; };
 static struct sa_cmp_ctx g_sa_ctx;
 
-static int sa_cmp_idx(int i, int j) {
-  int ri, rj;
+static int32_t sa_cmp_idx(int32_t i, int32_t j) {
+  int32_t ri, rj;
   if (g_sa_ctx.rank[i] != g_sa_ctx.rank[j])
     return g_sa_ctx.rank[i] - g_sa_ctx.rank[j];
-  ri = (i + (int)g_sa_ctx.k < (int)g_sa_ctx.n) ? g_sa_ctx.rank[i + g_sa_ctx.k] : -1;
-  rj = (j + (int)g_sa_ctx.k < (int)g_sa_ctx.n) ? g_sa_ctx.rank[j + g_sa_ctx.k] : -1;
+  ri = (i + (int32_t)g_sa_ctx.k < (int32_t)g_sa_ctx.n) ? g_sa_ctx.rank[i + g_sa_ctx.k] : -1;
+  rj = (j + (int32_t)g_sa_ctx.k < (int32_t)g_sa_ctx.n) ? g_sa_ctx.rank[j + g_sa_ctx.k] : -1;
   return ri - rj;
 }
 
 static int sa_qsort_cmp(const void * a, const void * b) {
-  int i = *(const int *) a, j = *(const int *) b;
-  return sa_cmp_idx(i, j);
+  int32_t d = sa_cmp_idx(*(const int32_t *)a, *(const int32_t *)b);
+  return (d > 0) - (d < 0);
 }
 
-static int saca(const byte * s, size_t n, int * sa, int * rank, int * tmp) {
+static int saca(const byte * s, size_t n, int32_t * sa, int32_t * rank, int32_t * tmp) {
   size_t i;
   if (!n)
     return 0;
   for (i = 0; i < n; ++i) {
-    sa[i] = (int)i;  rank[i] = (int)s[i];
+    sa[i] = (int32_t)i;  rank[i] = (int32_t)s[i];
   }
   for (g_sa_ctx.k = 1;; g_sa_ctx.k <<= 1) {
     g_sa_ctx.rank = rank;  g_sa_ctx.n = n;
@@ -82,7 +133,7 @@ struct match_choice { uint32_t len;  uint16_t off; };
 struct parse_choice { uint32_t lit, mlen;  uint16_t off; };
 
 static int longest_matches(const byte * src, size_t n, struct match_choice * mch) {
-  int * sa, * rank, * tmp, * inv;
+  int32_t *sa, *rank, *tmp, *inv;
   size_t i;
   if (!n)
     return 0;
@@ -95,17 +146,18 @@ static int longest_matches(const byte * src, size_t n, struct match_choice * mch
     return -1;
   }
   for (i = 0; i < n; ++i)
-    inv[sa[i]] = (int)i;
+    inv[sa[i]] = (int32_t)i;
   for (i = 0; i < n; ++i) {
-    int r = inv[i], d, rr;
+    int32_t r = inv[i], rr;
+    int d;
     size_t best_len = 0;
     uint16_t best_off = 0;
     for (d = -LIMLZ_SA_NEIGHBORS; d <= LIMLZ_SA_NEIGHBORS; ++d) {
       size_t j, l, off;
       if (!d)
         continue;
       rr = r + d;
-      if (rr < 0 || rr >= (int)n)
+      if (rr < 0 || rr >= (int32_t)n)
         continue;
       j = (size_t)sa[rr];
       if (j >= i)
@@ -218,6 +270,7 @@ static size_t limlzpack(void * dst, size_t dstcap, const void * srcv, size_t src
     dp[i] = best_cost;  pick[i].lit = best_lit;
     pick[i].mlen = best_len;  pick[i].off = best_off;
   }
+  int terminated = 0;
   for (i = 0; i < srcsz; ) {
     byte * tokenp;
     size_t lit = pick[i].lit, ml = pick[i].mlen;
@@ -241,6 +294,7 @@ static size_t limlzpack(void * dst, size_t dstcap, const void * srcv, size_t src
     i += lit;
     if (i >= srcsz) {
       *tokenp = (byte)(token_hi << 3);
+      terminated = 1;
       break;
     }
     unsigned mode_bit = (off > 255) ? 1u : 0u;
@@ -260,6 +314,14 @@ static size_t limlzpack(void * dst, size_t dstcap, const void * srcv, size_t src
       goto fail;
     i += ml;
   }
+  /* A match-ended parse leaves no trailing token; the decompressor keys
+   * termination off a zero-or-more-byte literal copy reaching ipe, so always
+   * emit a final lit=0 token when the main loop didn't already. */
+  if (!terminated) {
+    if (out >= out_end)
+      goto fail;
+    *out++ = 0;
+  }
   free(mch);  free(pick);  free(bestm);  free(dp);
   return (size_t)(out - dstp);
 fail:
@@ -285,6 +347,11 @@ static uint32_t crc32_nibble(const byte *data, size_t len) {
 }
 
 int main(int argc, char *argv[]) {
+#ifndef __BYTE_ORDER__
+  uint32_t endcheck = 0x12345678;
+  uint8_t endbyte = *((uint8_t *)&endcheck);
+  bigendian = endbyte == 0x12;
+#endif
   if (argc != 3) {
     fprintf(stderr, "? %s <input> <output>\n", argv[0]);  return 1;
   }
@@ -308,7 +375,7 @@ int main(int argc, char *argv[]) {
   if (!outsz) {
     fprintf(stderr, "? limlzpack\n");  return 1;
   }
-  uint32_t crc = crc32_nibble(inbuf, insz);
+  uint32_t crc = ENDSWAP(crc32_nibble(inbuf, insz));
   fwrite(&crc, sizeof(crc), 1, fout);
   fwrite(outbuf, 1, outsz, fout);
   fclose(fout);