ruby : transcribe without GVL, accept more MemoryViews, Windows support, fix memory size report, improve document (ggml-org#3775)

* Change MemoryView example using NDAV

* Add note on audio attributes for #full and #full_parallel

* Support more variants of MemoryView

* Use IO.popen instead of Kernel.` for Windows compatibility

* Use cmake's -C option instead of multiple -D options

* Fix memsize calculation

* Remove unused argument

* Add is_interrupted field to abort callback container

* Fix RBS syntax

* Address document comment for RDoc

* Add .document for RDoc

* Add .rdoc_options

* Run #full without GVL

* Initialize callbacks with nil

* Specify implicity Whisper::Params to distinguish from Whisper::Context::Params

* Run callbacks without GVL

* Call log callback with GVL

* Run full_parallel without GVL

* Run transcribe without GVL

* Fix ruby_whisper_lock_gvl and ruby_whisper_unlock_gvl

* Fix return value of encoder_begin_callback

* Report GVL unlocking from transcribe

* Remove unused interface

* Restore overload of full_parallel

* Close process

* Fix struct name

* Make is_without_gvl thread local

* Use rb_thread_call_with_gvl instead of global variable

* Retrieve instance variable in GVL

* Narrow acceptable MemoryView format

* Fix option cache path

* Reduce files in package

* Use append_cflags

* Add ext/*.rb to task dependencies

* Use copy instead of cp

* Make TestPackage more portable

* Patch for lower version Ruby

* Make build scripts more portable

* Add Windows support

* Don't raise exceptions

Author: KitaitiMakoto

bindings/ruby/.document

@@ -0,0 +1,3 @@
+README.md
+LICENSE
+sig

bindings/ruby/.rdoc_options

@@ -0,0 +1,2 @@
+title: whispercpp
+main_page: README.md

bindings/ruby/README.md

@@ -360,7 +360,7 @@ Whisper::Context.new("base")
 
 ### Low-level API to transcribe ###
 
-You can also call `Whisper::Context#full` and `#full_parallel` with a Ruby array as samples. Although `#transcribe` with audio file path is recommended because it extracts PCM samples in C++ and is fast, `#full` and `#full_parallel` give you flexibility.
+You can also call `Whisper::Context#full` and `#full_parallel` with a Ruby array as samples. Although `#transcribe` with audio file path is recommended because it extracts PCM samples in C++ and is fast, `#full` and `#full_parallel` give you flexibility. Unlike `#transcribe`, these methods requires 16,000 Hz, 32-bit float audio.
 
 ```ruby
 require "whisper"
@@ -383,16 +383,16 @@ If you can prepare audio data as C array and export it as a MemoryView, whisperc
 
 ```ruby
 require "torchaudio"
-require "arrow-numo-narray"
+require "ndav/torch/tensor"
 require "whisper"
 
 waveform, sample_rate = TorchAudio.load("test/fixtures/jfk.wav")
-# Convert Torch::Tensor to Arrow::Array via Numo::NArray
-samples = waveform.squeeze.numo.to_arrow.to_arrow_array
+# Convert Torch::Tensor to NDAV
+samples = waveform.squeeze.to_ndav
 
 whisper = Whisper::Context.new("base")
 whisper
-  # Arrow::Array exports MemoryView
+  # NDAV exports MemoryView
   .full(Whisper::Params.new, samples)
 ```
 

bindings/ruby/Rakefile

@@ -16,7 +16,7 @@ EXTSOURCES.each do |src|
   file src
   directory dir
   file dest => [src, dir] do |t|
-    cp t.source, t.name
+    copy t.source, t.name
   end
   SOURCES.include dest
 end
@@ -34,7 +34,7 @@ LIB_NAME = "whisper".ext(RbConfig::CONFIG["DLEXT"])
 SO_FILE = File.join("ext", LIB_NAME)
 LIB_FILE = File.join("lib", LIB_NAME)
 
-file "ext/Makefile" => SRC + ["ext/extconf.rb"] + SOURCES do |t|
+file "ext/Makefile" => SRC + SOURCES + FileList["ext/*.rb"] do |t|
   chdir "ext" do
     ruby "extconf.rb"
   end

bindings/ruby/ext/dependencies.rb

@@ -22,23 +22,25 @@ def libs
       else
         nil
       end
-    }.reverse.collect {|lib| "lib#{lib}.a"}
+    }.reverse.collect {|lib| "#{prefix(lib)}#{lib}.#{RbConfig::CONFIG['LIBEXT']}"}
   end
 
   def to_s
     libs.join(" ")
   end
 
+  def local_libs
+    to_s
+  end
+
   private
 
   def dot_path
     File.join(__dir__, "build", "whisper.cpp.dot")
   end
 
   def generate_dot
-    args = ["-S", "sources", "-B", "build", "--graphviz", dot_path, "-D", "BUILD_SHARED_LIBS=OFF"]
-    args << @options.to_s unless @options.to_s.empty?
-    system @cmake, *args, exception: true
+    system @cmake, "-S", "sources", "-B", "build", *@options.graphviz_cmake_args, "--graphviz", dot_path, *@options, exception: true
   end
 
   def parse_dot
@@ -59,6 +61,10 @@ def parse_dot
     end
   end
 
+  def prefix(lib)
+    "lib"
+  end
+
   def tsort_each_node
     @nodes.each_key do |node|
       yield node

bindings/ruby/ext/dependencies_for_windows.rb

@@ -0,0 +1,17 @@
+require_relative "dependencies"
+
+class DependenciesForWindows < Dependencies
+  def local_libs
+    libs.collect {|lib| %|"#{lib_path(lib)}"|}.join(" ")
+  end
+
+  private
+
+  def prefix(lib)
+    lib.start_with?("ggml") ? "" : "lib"
+  end
+
+  def lib_path(lib)
+    File.join(__dir__, lib).tr("\\", "/")
+  end
+end

bindings/ruby/ext/extconf.rb

@@ -1,23 +1,35 @@
 require "mkmf"
-require_relative "options"
-require_relative "dependencies"
+
+if RUBY_PLATFORM.match? /mswin|mingw|ucrt/
+  require_relative "options_for_windows"
+  require_relative "dependencies_for_windows"
+
+  Opts = OptionsForWindows
+  Deps = DependenciesForWindows
+else
+  require_relative "options"
+  require_relative "dependencies"
+
+  Opts = Options
+  Deps = Dependencies
+end
 
 cmake = find_executable("cmake") || abort
-options = Options.new(cmake).to_s
+options = Opts.new(cmake)
 have_library("gomp") rescue nil
-libs = Dependencies.new(cmake, options).to_s
+libs = Deps.new(cmake, options)
 
-$CFLAGS << " -O3 -march=native"
+append_cflags ["-O3", "-march=native"]
 $INCFLAGS << " -Isources/include -Isources/ggml/include -Isources/examples"
-$LOCAL_LIBS << " #{libs}"
+$LOCAL_LIBS << " #{libs.local_libs}"
 $cleanfiles << " build #{libs}"
 
 create_makefile "whisper" do |conf|
   conf << <<~EOF
     $(TARGET_SO): #{libs}
     #{libs}: cmake-targets
     cmake-targets:
-    #{"\t"}#{cmake} -S sources -B build -D BUILD_SHARED_LIBS=OFF -D CMAKE_ARCHIVE_OUTPUT_DIRECTORY=#{__dir__} -D CMAKE_POSITION_INDEPENDENT_CODE=ON #{options}
-    #{"\t"}#{cmake} --build build --config Release --target common whisper
+    #{"\t"}"#{cmake}" -S sources -B build #{options}
+    #{"\t"}"#{cmake}" --build build --config Release --target common whisper
   EOF
 end

bindings/ruby/ext/options.rb

@@ -1,26 +1,36 @@
+require "fileutils"
+
 class Options
   def initialize(cmake="cmake")
     @cmake = cmake
     @options = {}
 
     configure
+    write_cache_file
+  end
+
+  def to_a
+    [
+      "-D", "BUILD_SHARED_LIBS=OFF",
+      "-D", "WHISPER_BUILD_TESTS=OFF",
+      "-D", "CMAKE_ARCHIVE_OUTPUT_DIRECTORY=#{__dir__}",
+      "-D", "CMAKE_POSITION_INDEPENDENT_CODE=ON",
+      "-C", cache_path
+    ]
   end
 
   def to_s
-    @options
-      .reject {|name, (type, value)| value.nil?}
-      .collect {|name, (type, value)| "-D #{name}=#{value == true ? "ON" : value == false ? "OFF" : value.shellescape}"}
-      .join(" ")
+    command_line(*to_a)
   end
 
-  def cmake_options
-    return @cmake_options if @cmake_options
+  def graphviz_cmake_args
+    []
+  end
 
-    output = nil
-    Dir.chdir __dir__ do
-      output = `#{@cmake.shellescape} -S sources -B build -L`
-    end
-    @cmake_options = output.lines.drop_while {|line| line.chomp != "-- Cache values"}.drop(1)
+  private
+
+  def cmake_options
+    @cmake_options ||= cmake_options_output.lines.drop_while {|line| line.chomp != "-- Cache values"}.drop(1)
                        .filter_map {|line|
                          option, value = line.chomp.split("=", 2)
                          name, type = option.split(":", 2)
@@ -34,7 +44,11 @@ def cmake_options
                        }.to_h
   end
 
-  private
+  def cmake_options_output
+    Dir.chdir(__dir__) do
+      IO.popen([@cmake, "-S", "sources", "-B", "build", "-L"]) {|io| io.read}
+    end
+  end
 
   def configure
     cmake_options.each_pair do |name, (type, default_value)|
@@ -74,12 +88,38 @@ def option_name(name)
 
   def enabled?(option)
     op = @options[option]
-    raise "Option not exist: #{option}" unless op
-    raise "Option not boolean: #{option}(#{op[0]})" unless op[0] == "BOOL"
+    return false unless op
+    return false unless op[0] == "BOOL"
     if op[1].nil?
       cmake_options[option][1]
     else
       op[1]
     end
   end
+
+  def cache_path
+    File.join(__dir__, "sources", "Options.cmake")
+  end
+
+  def write_cache_file
+    FileUtils.mkpath File.dirname(cache_path)
+    File.open cache_path, "w" do |file|
+      @options.reject {|name, (type, value)| value.nil?}.each do |name, (type, value)|
+        line = "set(CACHE{%<name>s} TYPE %<type>s FORCE VALUE %<value>s)" % {
+          name:,
+          type:,
+          value: value == true ? "ON" : value == false ? "OFF" : escape_cmake(value)
+        }
+        file.puts line
+      end
+    end
+  end
+
+  def escape_cmake(str)
+    str.gsub(/[\\"]/, '\\\\\&')
+  end
+
+  def command_line(*args)
+    args.collect {|arg| %|"#{arg.to_s.gsub(/[\\"]/, '\\\\\&')}"|}.join(" ")
+  end
 end

bindings/ruby/ext/options_for_windows.rb

@@ -0,0 +1,51 @@
+require_relative "options"
+
+class OptionsForWindows < Options
+  def to_s
+    command_line(*generator_args, *to_a)
+  end
+
+  def graphviz_cmake_args
+    generator_args
+  end
+
+  private
+
+  def arm?
+    RbConfig::CONFIG["host_cpu"].to_s.downcase.match?(/\A(?:arm64|aarch64)\z/)
+  end
+
+  def cmake_options_output
+    Dir.chdir(__dir__) do
+      IO.popen([@cmake, "-S", "sources", "-B", "build", *generator_args, "-L"]) {|io| io.read}
+    end
+  end
+
+  def generator_args
+    generator = cmake_generator
+    ["-G", generator] if generator && !generator.empty?
+  end
+
+  def cmake_generator
+    return @cmake_generator if defined?(@cmake_generator)
+
+    generator = ENV["CMAKE_GENERATOR"]
+    abort "CMAKE_GENERATOR=#{generator} is unsupported for mingw/ucrt Ruby" if visual_studio_generator_name?(generator)
+    return @cmake_generator = generator unless generator.nil? || generator.empty?
+
+    ninja = find_executable("ninja")
+    return @cmake_generator = "Ninja" if ninja
+
+    make = find_executable("make")
+    return @cmake_generator = "MSYS Makefiles" if make
+
+    mingw32_make = find_executable("mingw32-make")
+    return @cmake_generator = "MinGW Makefiles" if mingw32_make
+
+    @cmake_generator = nil
+  end
+
+  def visual_studio_generator_name?(generator)
+    generator && generator.start_with?("Visual Studio")
+  end
+end

bindings/ruby/ext/ruby_whisper.c

@@ -29,6 +29,7 @@ ID id_cache;
 ID id_n_processors;
 
 static bool is_log_callback_finalized = false;
+static bool is_ruby_log_callback_present = false;
 
 // High level API
 extern VALUE ruby_whisper_segment_allocate(VALUE klass);
@@ -106,18 +107,43 @@ static VALUE ruby_whisper_s_finalize_log_callback(VALUE self, VALUE id) {
   return Qnil;
 }
 
+typedef struct {
+  int level;
+  const char * buffer;
+} call_log_callbacks_args;
+
+static void*
+call_log_callbacks(void *v_args) {
+  VALUE log_callback = rb_iv_get(mWhisper, "log_callback");
+  if (NIL_P(log_callback)) {
+    return NULL;
+  }
+
+  call_log_callbacks_args *args = (call_log_callbacks_args *)v_args;
+  VALUE user_data = rb_iv_get(mWhisper, "user_data");
+  rb_funcall(log_callback, id_call, 3, INT2NUM(args->level), rb_str_new2(args->buffer), user_data);
+
+  return NULL;
+}
+
 static void
 ruby_whisper_log_callback(enum ggml_log_level level, const char * buffer, void * user_data) {
   if (is_log_callback_finalized) {
     return;
   }
-  VALUE log_callback = rb_iv_get(mWhisper, "log_callback");
-  if (NIL_P(log_callback)) {
+  if (!is_ruby_log_callback_present) {
     return;
   }
 
-  VALUE udata = rb_iv_get(mWhisper, "user_data");
-  rb_funcall(log_callback, id_call, 3, INT2NUM(level), rb_str_new2(buffer), udata);
+  call_log_callbacks_args args = {
+    level,
+    buffer,
+  };
+  if (ruby_thread_has_gvl_p()) {
+    call_log_callbacks((void *)&args);
+  } else {
+    rb_thread_call_with_gvl(call_log_callbacks, (void *)&args);
+  }
 }
 
 /*
@@ -140,8 +166,10 @@ static VALUE ruby_whisper_s_log_set(VALUE self, VALUE log_callback, VALUE user_d
 
   if (NIL_P(log_callback)) {
     whisper_log_set(NULL, NULL);
+    is_ruby_log_callback_present = false;
   } else {
     whisper_log_set(ruby_whisper_log_callback, NULL);
+    is_ruby_log_callback_present = true;
   }
 
   return Qnil;