Upgrade to Eio 0.15 and OCaml 5.1.1

Author: SGrondin

DEVELOPMENT.md

@@ -5,7 +5,8 @@ From the root of the repo:
 ```sh
 brew install opam libomp llvm
 
-opam switch create . ocaml-variants.5.0.0+options --deps-only -t
+opam switch create . ocaml-variants.5.1.1+options --no-install
+opam install . --deps-only -t
 
 # Remove old Flow version
 rm -rf flow && unlink src/flow_parser && unlink src/sedlex && unlink src/collections

src/cli/strings.ml

@@ -1,7 +1,7 @@
 open! Core
 open Eio.Std
 
-let version = "2.3.0"
+let version = "2.4.0"
 
 type counts = {
   vue: int ref;
@@ -33,7 +33,7 @@ type file_type =
 type traversal = {
   table: String.Set.t String.Table.t;
   counts: counts;
-  wp: Eio.Workpool.t;
+  wp: Eio.Executor_pool.t;
   slow_pug: bool;
   template_script: Vue.template_script;
   root: string;
@@ -96,19 +96,23 @@ let process_job ({ path; _ } as job) =
 
   Vue.collect_from_possible_scripts collector job.template_script
 
-let rec traverse ~fs ~stderr ({ slow_pug; template_script; counts; wp; _ } as traversal) directory =
+let rec traverse ~fs ~stderr ({ slow_pug; template_script; counts; _ } as traversal) directory =
   Eio.Path.with_open_dir Eio.Path.(fs / directory) Eio.Path.read_dir
   |> Fiber.List.iter (fun filename ->
        let path = Filename.concat directory filename in
-       match filename, Utils.Io.stat wp path with
+       let eio_path = Eio.Path.(fs / path) in
+       match filename, Eio.Path.kind ~follow:true eio_path with
        | "node_modules", _ -> ()
-       | _, { st_kind = S_DIR; _ } -> traverse ~fs ~stderr traversal path
-       | _, { st_kind = S_REG; _ } -> (
+       | _, `Directory -> traverse ~fs ~stderr traversal path
+       | _, `Regular_file -> (
          match file_type_of_filename filename with
          | None -> ()
          | Some file_type ->
-           let job = { file_type; eio_path = Eio.Path.(fs / path); path; template_script; slow_pug } in
-           let collector = Eio.Workpool.submit_exn traversal.wp (fun () -> process_job job) in
+           let job = { file_type; eio_path; path; template_script; slow_pug } in
+           let collector =
+             Eio.Executor_pool.submit_exn traversal.wp ~weight:Utils.Io.traversal_jobs_weight (fun () ->
+               process_job job )
+           in
            let count =
              match job.file_type with
              | JS -> counts.js
@@ -134,7 +138,7 @@ let main env options = function
       let languages = Vue.parse ~path ~slow_pug flow in
       Vue.debug_template ~stdout ~path languages template_script lang
     | Pug, ".pug" -> (
-      let source = Utils.Io.load_flow flow in
+      let source = Eio.Flow.read_all flow in
       let slow_parse () =
         let collector = Utils.Collector.create ~path in
         Quickjs.extract_to_collector collector Pug source;
@@ -164,27 +168,22 @@ let main env options = function
     | _ -> Eio.Flow.copy_string (sprintf "Nothing to do for file [%s]\n" path) stdout )
 | Run ->
   let overall_time = Utils.Timing.start () in
-  Switch.run @@ fun sw ->
   let outdir = options.outdir in
   if List.is_empty options.targets then failwith "Please specify at least one directory";
   let fs = Eio.Stdenv.fs env in
   let stdout = Eio.Stdenv.stdout env in
-  let sys_wp =
-    Eio.Workpool.create ~sw ~domain_count:Utils.Io.num_systhreads ~domain_concurrency:1
-      (Eio.Stdenv.domain_mgr env)
-  in
   (* Check current directory *)
   let strings_dir_files =
     let git_dir, strings_dir =
       Fiber.pair
-        (fun () -> Utils.Io.directory_exists sys_wp ".git")
-        (fun () -> Utils.Io.directory_exists sys_wp outdir)
+        (fun () -> Eio.Path.is_directory Eio.Path.(fs / ".git"))
+        (fun () -> Eio.Path.is_directory Eio.Path.(fs / outdir))
     in
     if not (git_dir || strings_dir) then failwith "This program must be run from the root of your project";
     match strings_dir with
     | true -> Eio.Path.with_open_dir Eio.Path.(fs / outdir) Eio.Path.read_dir
     | false ->
-      Utils.Io.mkdir_p env sys_wp ~dir_name:outdir ~perms:0o751;
+      Eio.Path.mkdirs ~exists_ok:true ~perm:0o751 Eio.Path.(fs / outdir);
       []
   in
 
@@ -194,10 +193,7 @@ let main env options = function
     let table = String.Table.create () in
     let counts = { vue = ref 0; pug = ref 0; html = ref 0; js = ref 0; ts = ref 0 } in
     Switch.run @@ fun sw ->
-    let wp =
-      Eio.Workpool.create ~sw ~domain_count:Utils.Io.num_cores
-        ~domain_concurrency:Utils.Io.traversal_jobs_per_core (Eio.Stdenv.domain_mgr env)
-    in
+    let wp = Eio.Executor_pool.create ~sw ~domain_count:Utils.Io.num_cores (Eio.Stdenv.domain_mgr env) in
     options.targets
     |> Fiber.List.iter (fun directory ->
          let root_slash, root_noslash =
@@ -242,9 +238,10 @@ let main env options = function
         | Some "english" -> ()
         | Some language -> (
           let path = Filename.concat outdir filename in
-          match Utils.Io.stat sys_wp path with
-          | { st_kind = S_REG; _ } ->
-            let other = Eio.Path.with_open_in Eio.Path.(fs / path) (Parsing.Strings.parse ~path) in
+          let eio_path = Eio.Path.(fs / path) in
+          match Eio.Path.kind ~follow:true eio_path with
+          | `Regular_file ->
+            let other = Eio.Path.with_open_in eio_path (Parsing.Strings.parse ~path) in
             Generate.write_other ~fs ~stdout ~version ~outdir ~language english other
           | _ -> () )
         | None -> ())

src/quickjs/dune

@@ -53,7 +53,7 @@
 (rule
   (targets libomp.a)
   (action (bash "
-    cp /usr/local/Cellar/libomp/16.0.5/lib/libomp.a . &> /dev/null \
+    cp /usr/local/Cellar/libomp/17.0.6/lib/libomp.a . &> /dev/null \
     || cp /usr/lib/libgomp.a libomp.a
   "))
   (mode standard)

src/quickjs/quickjs.ml

@@ -14,33 +14,12 @@ external stub_init_contexts : int -> (unit, string) Result.t = "stub_init_contex
 external stub_extract : int -> string -> fn_name:string -> (string array * string array, string) Result.t
   = "stub_extract"
 
-module Pool : sig
-  type t
-
-  val create : int -> t
-
-  val with_pool : t -> f:(int -> 'a) -> 'a
-end = struct
-  type t = int Eio.Stream.t
-
-  let create n =
-    let stream = Eio.Stream.create n in
-    for i = 0 to n - 1 do
-      Eio.Stream.add stream i
-    done;
-    stream
-
-  let with_pool stream ~f =
-    let id = Eio.Stream.take stream in
-    let result =
-      try f id with
-      | exn ->
-        Eio.Stream.add stream id;
-        raise exn
-    in
-    Eio.Stream.add stream id;
-    result
-end
+let pool =
+  let i = ref 0 in
+  Eio.Pool.create Utils.Io.num_js_workers @@ fun () ->
+  let x = !i in
+  incr i;
+  x
 
 let init_time = Atomic.make Int63.zero
 
@@ -57,7 +36,7 @@ let init_contexts =
 
       let time = time `Stop in
       Atomic.set init_time time;
-      Promise.resolve w (Pool.create num_js_workers);
+      Promise.resolve w pool;
       print_endline
         (sprintf
            !"✅ [%{Int63}ms] Initialized %d JS runtimes for TS and/or Pug processing\n"
@@ -101,7 +80,7 @@ let extract kind code =
     | Pug -> clean_pug code
   in
   let fn_name = fn_name_of_kind kind in
-  Pool.with_pool pool ~f:(fun id -> stub_extract id code ~fn_name)
+  Eio.Pool.use pool (fun id -> stub_extract id code ~fn_name)
 
 let extract_to_collector (collector : Utils.Collector.t) kind code =
   match extract kind code with

src/utils/io.ml

@@ -6,49 +6,4 @@ let num_js_workers = 4
 
 let num_cores = 4
 
-let traversal_jobs_per_core = 4
-
-let num_systhreads = 2
-
-let stat wp path : Core_unix.stats = Eio.Workpool.submit_exn wp (fun () -> Core_unix.stat path)
-
-let load_flow flow =
-  let open Eio in
-  (* Taken from [Eio.Path.load] *)
-  try
-    let size = File.size flow in
-    if Optint.Int63.(compare size (of_int Sys.max_string_length)) = 1 then raise @@ Fs.err File_too_large;
-    let buf = Cstruct.create (Optint.Int63.to_int size) in
-    let rec loop buf got =
-      match Flow.single_read flow buf with
-      | n -> loop (Cstruct.shift buf n) (n + got)
-      | exception End_of_file -> got
-    in
-    let got = loop buf 0 in
-    Cstruct.to_string ~len:got buf
-  with
-  | Exn.Io _ as ex ->
-    let bt = Stdlib.Printexc.get_raw_backtrace () in
-    Exn.reraise_with_context ex bt "loading flow"
-
-let directory_exists wp path =
-  match stat wp path with
-  | { st_kind = S_DIR; _ } -> true
-  | { st_kind = _; _ } -> failwithf "%s already exists, but is not a directory" path ()
-  | exception _ -> false
-
-let mkdir_p env wp ~dir_name ~perms:perm =
-  let (_ : string) =
-    Filename.parts dir_name
-    |> List.fold ~init:"" ~f:(fun acc part ->
-         match acc with
-         | "" -> part
-         | acc -> (
-           let path = Filename.concat acc part in
-           directory_exists wp path |> function
-           | true -> path
-           | false ->
-             Eio.Path.mkdir ~perm Eio.Path.(env#fs / path);
-             path ) )
-  in
-  ()
+let traversal_jobs_weight = 0.25

strings.opam

@@ -15,26 +15,25 @@ bug-reports: "https://github.com/okTurtles/strings/issues"
 pin-depends: [
   [ "angstrom-eio.0.15.0.custom" "git+https://github.com/SGrondin/angstrom.git#1a961940ffb746379b4bdc648bc194501b65348a" ]
   [ "angstrom.0.15.0.custom" "git+https://github.com/SGrondin/angstrom.git#1a961940ffb746379b4bdc648bc194501b65348a" ]
-  [ "eio.0.12.custom" "git+https://github.com/SGrondin/eio.git#37e0dc0d98bd6c18506a974c3ce2b81e89be30a9" ]
-  [ "eio_posix.0.12.custom" "git+https://github.com/SGrondin/eio.git#37e0dc0d98bd6c18506a974c3ce2b81e89be30a9" ]
-  [ "eio_linux.0.12.custom" "git+https://github.com/SGrondin/eio.git#37e0dc0d98bd6c18506a974c3ce2b81e89be30a9" ]
-  [ "eio_main.0.12.custom" "git+https://github.com/SGrondin/eio.git#37e0dc0d98bd6c18506a974c3ce2b81e89be30a9" ]
-  [ "SZXX.4.0.1.custom" "git+https://github.com/asemio/SZXX.git#4.0.1" ]
+  [ "eio.0.15.custom" "/Users/simongrondin/dev/eio" ]
+  [ "eio_posix.0.15.custom" "/Users/simongrondin/dev/eio" ]
+  [ "eio_linux.0.15.custom" "/Users/simongrondin/dev/eio" ]
+  [ "eio_main.0.15.custom" "/Users/simongrondin/dev/eio" ]
 ]
 depends: [
-  "ocaml" { = "5.0.0" }
+  "ocaml" { = "5.1.1" }
   "ocaml-option-flambda"
   "dune" { >= "2.8.0"}
 
   "ocamlformat" { = "0.25.1" & with-test }
   "ocaml-lsp-server" { with-test }
 
-  "eio_main" { = "0.12.custom" }
+  "eio_main" { = "0.15.custom" }
 
   "angstrom" { = "0.15.0.custom" }
   "angstrom-eio" { = "0.15.0.custom" }
-  "core" { >= "v0.15.0" }
-  "core_unix" { >= "v0.15.0" }
+  "core" { >= "v0.16.0" }
+  "core_unix" { >= "v0.16.0" }
   "ppx_deriving"
   "ppx_deriving_yojson"
   "ppx_gen_rec"
@@ -43,5 +42,5 @@ depends: [
   "yojson"
   "uuidm"
   "wtf8"
-  "SZXX" { = "4.0.1.custom" }
+  "SZXX" { >= "4.1.0" }
 ]