diff --git a/.gitignore b/.gitignore index 66fc22b56..538ab7319 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ zig-out lightpanda.id /src/html5ever/target/ src/snapshot.bin +result diff --git a/build.zig.zon.nix b/build.zig.zon.nix new file mode 100644 index 000000000..0e27acc5a --- /dev/null +++ b/build.zig.zon.nix @@ -0,0 +1,65 @@ +# generated by zon2nix (https://github.com/nix-community/zon2nix) + +{ linkFarm, fetchzip, fetchgit }: + +linkFarm "zig-packages" [ + { + name = "N-V-__8AABDOXwDW4TLrTiydikRCN2ym9hJI1GKGR09ZLBvY"; + path = fetchzip { + url = "https://chromium.googlesource.com/chromium/tools/depot_tools.git/+archive/4ce8ba39a3488397a2d1494f167020f21de502f3.tar.gz"; + stripRoot = false; + hash = "sha256-WTzjmLFjh1yDDEvYE7Qfx8aBxMLdATx14+Jprwh8ZgQ="; + }; + } + { + name = "N-V-__8AADGOTAg2qy2JMjVSleelXVWZhf9UTIjkuioQsv9-"; + path = fetchgit { + url = "https://github.com/google/boringssl"; + rev = "535cc391915c37912ff9b3edb0cff9b3c5c77db0"; + hash = "sha256-rSmfL4FC9QgIsQJWHwGlHuAR1ntHrK271A392FacQrs="; + }; + } + { + name = "N-V-__8AAJ2cNgAgfBtAw33Bxfu1IWISDeKKSr3DAqoAysIJ"; + path = fetchzip { + url = "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz"; + hash = "sha256-tKLefmGqB2HVUMILJ7Gqa0v9GH7fqv64d5S5kK1tcYU="; + }; + } + { + name = "N-V-__8AAJudKgCQCuIiH6MJjAiIJHfg_tT_Ew-0vZwVkCo_"; + path = fetchzip { + url = "https://github.com/google/brotli/archive/028fb5a23661f123017c060daa546b55cf4bde29.tar.gz"; + hash = "sha256-kl8ZHt71v17QR2bDP+ad/5uixf+GStEPLQ5ooFoC5i8="; + }; + } + { + name = "N-V-__8AAL15vQCI63ZL6Zaz5hJg6JTEgYXGbLnMFSnf7FT3"; + path = fetchzip { + url = "https://github.com/nghttp2/nghttp2/releases/download/v1.68.0/nghttp2-1.68.0.tar.gz"; + hash = "sha256-vNMN3b2tPRF0XhBlbrwZxZiLop7QDIbVWCZmEmlQb/I="; + }; + } + { + name = "N-V-__8AALp9QAGn6CCHZ6fK_FfMyGtG824LSHYHHasM3w-y"; + path = fetchzip { + url = "https://github.com/curl/curl/releases/download/curl-8_18_0/curl-8.18.0.tar.gz"; + hash = "sha256-//iOgGs/eZqciVZdgaf+PrmiFb3ACH4aTICVD4/Hkbg="; + }; + } + { + name = "boringssl-0.1.0-VtJeWehMAAA4RNnwRnzEvKcS9rjsR1QVRw1uJrwXxmVK"; + path = fetchgit { + url = "https://github.com/Syndica/boringssl-zig"; + rev = "c53df00d06b02b755ad88bbf4d1202ed9687b096"; + hash = "sha256-I6ST8fEQ35At/cRxjXJGxDiqZHhoJnW9n7GrhpO6RmE="; + }; + } + { + name = "v8-0.0.0-xddH6_F3BAAiFvKY6R1H-gkuQlk19BkDQ0--uZuTrSup"; + path = fetchzip { + url = "https://github.com/lightpanda-io/zig-v8-fork/archive/refs/tags/v0.3.4.tar.gz"; + hash = "sha256-86P24bN0n2zWBzSIdFz8rLIR8SARUtUsfHD11Psdrv0="; + }; + } +] diff --git a/flake.lock b/flake.lock index dcf837fa2..8fda513c5 100644 --- a/flake.lock +++ b/flake.lock @@ -37,6 +37,43 @@ "type": "github" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "zon2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1741352980, + "narHash": "sha256-+u2UunDA4Cl5Fci3m7S643HzKmIDAe+fiXrLqYsR2fs=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "f4330d22f1c5d2ba72d3d22df5597d123fdb60a9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -73,6 +110,24 @@ "type": "github" } }, + "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, + "locked": { + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "gitignore": { "inputs": { "nixpkgs": [ @@ -110,13 +165,30 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1742288794, + "narHash": "sha256-Txwa5uO+qpQXrNG4eumPSD+hHzzYi/CdaM80M9XRLCo=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "b6eaf97c6960d97350c584de1b6dcff03c9daf42", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "fenix": "fenix", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "zigPkgs": "zigPkgs", - "zlsPkg": "zlsPkg" + "zlsPkg": "zlsPkg", + "zon2nix": "zon2nix" } }, "rust-analyzer-src": { @@ -166,6 +238,44 @@ "type": "github" } }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "zig-overlay": { + "inputs": { + "flake-compat": "flake-compat_2", + "flake-utils": "flake-utils_3", + "nixpkgs": [ + "zon2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1770165742, + "narHash": "sha256-q7nu9F2ZtmIU9BMarY8BLIEfiwPwrjXGXLvOdFSvQP4=", + "owner": "mitchellh", + "repo": "zig-overlay", + "rev": "364ec9318bc70fc1ec084cb41e80e90d6770dd2b", + "type": "github" + }, + "original": { + "owner": "mitchellh", + "repo": "zig-overlay", + "type": "github" + } + }, "zigPkgs": { "inputs": { "flake-compat": "flake-compat", @@ -212,6 +322,26 @@ "repo": "zls", "type": "github" } + }, + "zon2nix": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs_2", + "zig-overlay": "zig-overlay" + }, + "locked": { + "lastModified": 1770337306, + "narHash": "sha256-Nxe1MvrNNc9LN34NJeAQ91SDE6mW710G2xRSmI0hq4M=", + "owner": "nix-community", + "repo": "zon2nix", + "rev": "8c2ef6d376727cd5dc768a851ccded47161e65fb", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "zon2nix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 330bbdf04..b764f1092 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,8 @@ }; flake-utils.url = "github:numtide/flake-utils"; + + zon2nix.url = "github:nix-community/zon2nix"; }; outputs = @@ -26,13 +28,19 @@ zlsPkg, fenix, flake-utils, + zon2nix, ... }: flake-utils.lib.eachDefaultSystem ( system: let + zigVersion = "0.15.2"; + zigV8Version = "v0.3.4"; + v8Version = "14.0.365.4"; + cargoRoot = "src/html5ever"; + overlays = [ - (final: prev: { + (_final: prev: { zigpkgs = zigPkgs.packages.${prev.system}; zls = zlsPkg.packages.${prev.system}.default; }) @@ -44,6 +52,21 @@ rustToolchain = fenix.packages.${system}.stable.toolchain; + zon2nixScript = pkgs.writeShellApplication { + name = "zon2nix"; + runtimeInputs = + with pkgs; + [ + gawk + ripgrep + ] + ++ [ zon2nix.packages.${system}.default ]; + text = # bash + '' + exec bash "$PWD/zon2nix.sh" + ''; + }; + # We need crtbeginS.o for building. crtFiles = pkgs.runCommand "crt-files" { } '' mkdir -p $out/lib @@ -55,9 +78,11 @@ name = "fhs-shell"; multiArch = true; targetPkgs = - pkgs: with pkgs; [ - # Build Tools - zigpkgs."0.15.2" + pkgs: + with pkgs; + [ + # Build tools + zigpkgs.${zigVersion} zls rustToolchain python3 @@ -65,20 +90,131 @@ cmake gperf - # GCC + # Toolchain/runtime pieces Zig expects during builds gcc gcc.cc.lib crtFiles - # Libaries + # Libraries expat.dev glib.dev glibc.dev zlib - ]; + ] + ++ [ zon2nixScript ]; + }; + + # regenerate with `nix run .#zon2nix` + zigDeps = import ./build.zig.zon.nix { inherit (pkgs) linkFarm fetchgit fetchzip; }; + zigTarget = if pkgs.stdenv.hostPlatform.isAarch64 then "aarch64-linux-gnu" else "x86_64-linux-gnu"; + + cargoDeps = pkgs.rustPlatform.fetchCargoVendor { + src = ./.; + inherit cargoRoot; + hash = "sha256-2eUx3gG6ufaEuHESGq33UGMNIN2W4LF96/QjyUFIops="; }; + + prebuiltV8 = + if pkgs.stdenv.hostPlatform.isLinux then + let + arch = if pkgs.stdenv.hostPlatform.isAarch64 then "aarch64" else "x86_64"; + in + pkgs.fetchurl { + url = "https://github.com/lightpanda-io/zig-v8-fork/releases/download/${zigV8Version}/libc_v8_${v8Version}_linux_${arch}.a"; + hash = + if pkgs.stdenv.hostPlatform.isAarch64 then + "sha256-Zg5/e4c4r4zIQ7D7vCnnDPyxwU8H05HcIqfMSHFFE6k=" + else + "sha256-lu0iuSuV42vch+92sKlcw3NjC2ko4XljgyOMTCnQeKk="; + } + else + null; + + prebuiltV8Arg = pkgs.lib.optionalString (prebuiltV8 != null) "-Dprebuilt_v8_path=${prebuiltV8}"; in { + packages = rec { + default = lightpanda; + lightpanda = pkgs.stdenv.mkDerivation { + pname = "lightpanda"; + version = "unstable"; + src = ./.; + + nativeBuildInputs = [ + pkgs.rustPlatform.cargoSetupHook + pkgs.zigpkgs.${zigVersion} + rustToolchain + pkgs.python3 + pkgs.pkg-config + pkgs.cmake + pkgs.gperf + pkgs.gcc + pkgs.patchelf + ]; + + buildInputs = [ + pkgs.expat + pkgs.glib + pkgs.glibc.dev + pkgs.zlib + pkgs.gcc.cc.lib + crtFiles + ]; + + inherit cargoDeps cargoRoot; + + dontConfigure = true; + + buildPhase = '' + runHook preBuild + + export HOME="$TMPDIR/home" + export XDG_CACHE_HOME="$TMPDIR/.cache" + export CARGO_HOME="$TMPDIR/cargo" + export RUSTUP_HOME="$TMPDIR/rustup" + export ZIG_GLOBAL_CACHE_DIR="$TMPDIR/zig-cache" + export ZIG_LIBC="$TMPDIR/zig-libc.conf" + export PATH=${pkgs.stdenv.cc}/bin:$PATH + export CC=cc + export CXX=c++ + + mkdir -p "$HOME" "$XDG_CACHE_HOME" "$CARGO_HOME" "$RUSTUP_HOME" "$ZIG_GLOBAL_CACHE_DIR" + zig libc > "$ZIG_LIBC" + ln -s ${zigDeps} "$ZIG_GLOBAL_CACHE_DIR/p" + + cp -r "$src" source + chmod -R u+w source + cd source + + zig build install -Dtarget=${zigTarget} -Doptimize=ReleaseFast ${prebuiltV8Arg} + patchelf --set-interpreter ${pkgs.stdenv.cc.bintools.dynamicLinker} zig-out/bin/lightpanda-snapshot-creator + ./zig-out/bin/lightpanda-snapshot-creator src/snapshot.bin + zig build install -Dtarget=${zigTarget} -Doptimize=ReleaseFast -Dgit_commit=dev -Dsnapshot_path=../../snapshot.bin ${prebuiltV8Arg} + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p "$out/bin" + cp -v zig-out/bin/lightpanda "$out/bin/" + patchelf --set-interpreter ${pkgs.stdenv.cc.bintools.dynamicLinker} "$out/bin/lightpanda" + + runHook postInstall + ''; + + meta.mainProgram = "lightpanda"; + }; + }; + + apps = { + zon2nix = { + type = "app"; + program = pkgs.lib.getExe zon2nixScript; + }; + }; + devShells.default = fhs.env; } ); diff --git a/zon2nix.sh b/zon2nix.sh new file mode 100755 index 000000000..4c7c5478f --- /dev/null +++ b/zon2nix.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd -- "$(cd -- "$(dirname -- "$0")" && pwd)" + +tmp_in="$(mktemp)" +tmp_out="$(mktemp)" +trap 'rm -f "$tmp_in" "$tmp_out"' EXIT + +echo "generating build.zig.zon.nix from build.zig.zon..." + +zon2nix > "$tmp_in" + +echo "patching non-canonical urls in build.zig.zon.nix" + +# IMPORTANT: the program below assumes that build.zig.zon is correct and +# that ".url = ..." is placed above ".hash = ...", which is Zig's default +# behaviour. + +awk ' + # build mapping from dependency hash/name -> canonical url + # for build.zig.zon ONLY + FNR == NR { + # match for `.url = "..."` + # remember "..." zon_url (without quotation) + if (match($0, /^[[:space:]]*\.url[[:space:]]*=[[:space:]]*"([^"]+)"/, m)) { + zon_url = m[1] + } + # match for `.hash = "..."` for current `zon_url` + # map canonical url => hash value (without quotation) + if (match($0, /^[[:space:]]*\.hash[[:space:]]*=[[:space:]]*"([^"]+)"/, m) && zon_url != "") { + canon[m[1]] = zon_url + zon_url = "" + } + next + } + + function insert_after(line_no, text, i) { + for (i = n; i > line_no; i--) { + item[i + 1] = item[i] + } + item[line_no + 1] = text + n++ + } + + function flush( i, indent, m) { + # only rewrite if: + # - a `name = "..."` is found for this item, + # - a `url = "..."` is found with line number to patch, + # - and the canonical URL is not a git+ url, as zon2nix turns those into + # fetchgit-style entries + if (name in canon && url_line && canon[name] !~ /^git\+/) { + sub(/"[^"]+"/, "\"" canon[name] "\"", item[url_line]) + } + # chromium +archive tarballs are flat archives, so fetchzip must preserve + # their root layout with `striproot = false;`. + # if we are in a fetchzip item, the final url contains /+archive/, and the + # generated item does not already contain striproot, insert that line. + if (is_fetchzip && url_line && item[url_line] ~ /\/\+archive\// && !has_strip_root) { + indent = " " + if (match(item[url_line], /^([[:space:]]*)url[[:space:]]*=.*$/, m)) { + indent = m[1] + } + insert_after(url_line, indent "stripRoot = false;") + } + for (i = 1; i <= n; i++) { + print item[i] + delete item[i] + } + n = 0 + name = "" + url_line = 0 + is_fetchzip = 0 + has_strip_root = 0 + in_item = 0 + } + + # not inside a linkfarm item + !in_item { + # line just opening `{` => linkfarm item starts here => in item + if ($0 ~ /^[[:space:]]*\{[[:space:]]*$/) { + in_item = 1 + n = 1 + item[n] = $0 + } else { + # regular line + print + } + next + } + + # inside a linkfarm item + { + item[++n] = $0 + + # found `path = fetchzip {` inside this linkFarm item + if ($0 ~ /^[[:space:]]*path[[:space:]]*=[[:space:]]*fetchzip[[:space:]]*\{[[:space:]]*$/) { + is_fetchzip = 1 + } + # found linkfarm item `name = "..."` + if (match($0, /^[[:space:]]*name[[:space:]]*=[[:space:]]*"([^"]+)";/, m)) { + name = m[1] + } + # found link farm item `url = "..."` + if (match($0, /^[[:space:]]*url[[:space:]]*=[[:space:]]*"([^"]+)";/, m)) { + url_line = n + } + # remember whether stripRoot is already present so we do not insert it twice + if ($0 ~ /^[[:space:]]*stripRoot[[:space:]]*=[[:space:]]*false;[[:space:]]*$/) { + has_strip_root = 1 + } + # match for "}" => found end of linkfarm item + if ($0 ~ /^[[:space:]]*\}[[:space:]]*$/) { + flush() + } + } + + END { + if (in_item) { + flush() + } + } +' build.zig.zon "$tmp_in" > "$tmp_out" + +mv "$tmp_out" build.zig.zon.nix + +if rg -q 'release-assets\.githubusercontent\.com' build.zig.zon.nix; then + echo 'error: build.zig.zon.nix still contains redirected GitHub asset URLs' >&2 + exit 1 +fi + +echo 'updated build.zig.zon.nix'