🔀 Merge pull request #649 from ruby/add-support-for-non-synchronizing-literals

✨ Support `LITERAL+` and `LITERAL-` non-synchronizing literals (RFC7888)

Author: nevans

lib/net/imap.rb

@@ -450,16 +450,16 @@ module Net
   #
   # Although IMAP4rev2[https://www.rfc-editor.org/rfc/rfc9051] is not supported
   # yet, Net::IMAP supports several extensions that have been folded into it:
-  # +ENABLE+, +IDLE+, +MOVE+, +NAMESPACE+, +SASL-IR+, +UIDPLUS+, +UNSELECT+,
-  # <tt>STATUS=SIZE</tt>, and the fetch side of +BINARY+.
+  # +ENABLE+, +IDLE+, +LITERAL-+, +MOVE+, +NAMESPACE+, +SASL-IR+, +UIDPLUS+,
+  # +UNSELECT+, <tt>STATUS=SIZE</tt>, and the fetch side of +BINARY+.
   # Commands for these extensions are listed with the {Core IMAP
   # commands}[rdoc-ref:Net::IMAP@Core+IMAP+commands], above.
   #
   # >>>
   #   <em>The following are folded into +IMAP4rev2+ but are currently
   #   unsupported or incompletely supported by</em> Net::IMAP<em>: RFC4466
   #   extensions, +SEARCHRES+, +LIST-EXTENDED+, +LIST-STATUS+,
-  #   +LITERAL-+, and +SPECIAL-USE+.</em>
+  #   and +SPECIAL-USE+.</em>
   #
   # ==== RFC2087: +QUOTA+
   # +NOTE:+ Only the +STORAGE+ quota resource type is currently supported.
@@ -569,6 +569,15 @@ module Net
   # - Updates #store and #uid_store with the +unchangedsince+ modifier and adds
   #   the +MODIFIED+ ResponseCode to the tagged response.
   #
+  # ==== RFC7888: <tt>LITERAL+</tt>
+  # - Literal strings smaller than Config#max_non_synchronizing_literal bytes
+  #   are sent without waiting for the server's continuation request.
+  #
+  # ==== RFC7888: +LITERAL-+
+  # - Literal strings smaller than 4096 bytes or
+  #   Config#max_non_synchronizing_literal (whichever is smaller)
+  #   are sent without waiting for the server's continuation request.
+  #
   # ==== RFC8438: <tt>STATUS=SIZE</tt>
   # - Updates #status with the +SIZE+ status attribute.
   #

lib/net/imap/command_data.rb

@@ -4,6 +4,8 @@
 
 require_relative "errors"
 
+# :enddoc:
+
 module Net
   class IMAP < Protocol
 
@@ -77,12 +79,23 @@ def send_quoted_string(str)
       put_string('"' + str.gsub(/["\\]/, "\\\\\\&") + '"')
     end
 
-    def send_binary_literal(str, tag) = send_literal(str, tag, binary: true)
+    def send_binary_literal(*, **) = send_literal(*, **, binary: true)
 
-    def send_literal(str, tag = nil, binary: false)
+    # `non_sync` is an optional tri-state flag:
+    # * `true`  -> Force non-synchronizing `LITERAL+`/`LITERAL-` behavior.
+    #   TODO: raise or warn when capabilities don't allow non_sync.
+    # * `false` -> Force normal synchronizing literal behavior.
+    # * `nil`   -> (default) Currently behaves like `false` (will be dynamic).
+    def send_literal(str, tag = nil, binary: false, non_sync: nil)
       synchronize do
+        non_sync = non_sync_literal?(str.bytesize) if non_sync.nil?
         prefix = "~" if binary
-        put_string("#{prefix}{#{str.bytesize}}\r\n")
+        plus = "+" if non_sync
+        put_string("#{prefix}{#{str.bytesize}#{plus}}\r\n")
+        if non_sync
+          put_string(str)
+          return
+        end
         @continued_command_tag = tag
         @continuation_request_exception = nil
         begin
@@ -97,6 +110,13 @@ def send_literal(str, tag = nil, binary: false)
       end
     end
 
+    def non_sync_literal?(bytesize)
+      capabilities_cached? &&
+        bytesize <= config.max_non_synchronizing_literal &&
+        (capable?("LITERAL+") ||
+         bytesize <= 4096 && (capable?("IMAP4rev2") || capable?("LITERAL-")))
+    end
+
     def send_number_data(num)
       put_string(num.to_s)
     end
@@ -149,8 +169,14 @@ def send_data(imap, tag)
       end
     end
 
-    class Literal < CommandData # :nodoc:
-      def initialize(data:)
+    class Literal < Data.define(:data, :non_sync) # :nodoc:
+      def self.validate(...)
+        data = new(...)
+        data.validate
+        data
+      end
+
+      def initialize(data:, non_sync: nil)
         data = -String(data.to_str).b or
           raise DataFormatError, "#{self.class} expects string input"
         super
@@ -167,15 +193,15 @@ def validate
       end
 
       def send_data(imap, tag)
-        imap.__send__(:send_literal, data, tag)
+        imap.__send__(:send_literal, data, tag, non_sync:)
       end
     end
 
     class Literal8 < Literal # :nodoc:
       def validate = nil # all bytes are okay
 
       def send_data(imap, tag)
-        imap.__send__(:send_binary_literal, data, tag)
+        imap.__send__(:send_binary_literal, data, tag, non_sync:)
       end
     end
 

lib/net/imap/config.rb

@@ -281,6 +281,40 @@ def self.[](config)
         0.5r => true,
       }
 
+      # The maximum bytesize for sending non-synchronizing literals, when the
+      # server supports them.  To disable non-synchronizing literals, set the
+      # value to +-1+.
+      #
+      # Non-synchronizing literals are only sent when the server's
+      # capabilities[rdoc-ref:IMAP#capabilities] have been
+      # cached[rdoc-ref:IMAP#capabilities_cached?] and include either
+      # <tt>LITERAL+</tt> [RFC7888[https://www.rfc-editor.org/rfc/rfc7888]],
+      # <tt>LITERAL-</tt> [RFC7888[https://www.rfc-editor.org/rfc/rfc7888]], or
+      # +IMAP4rev2+ [RFC9051[https://www.rfc-editor.org/rfc/rfc9051]].
+      #
+      # For <tt>LITERAL+</tt>, this value is the only limit on whether a literal
+      # value is sent as non-synchronizing literals.  For <tt>LITERAL-</tt> and
+      # <tt>IMAP4rev2</tt>, non-synchronizing literals must also be smaller than
+      # +4096+ bytes.
+      #
+      # Non-synchronizing literals avoid the latency of waiting for the server
+      # to allow continuation.  However, if a client sends a non-synchronizing
+      # literal that is too large for the server, the server may need to close
+      # the connection.  Because <tt>LITERAL+</tt> does not directly indicate
+      # the server's limits, it's best to avoid sending very large
+      # non-synchronized literals.
+      #
+      # ==== Versioned Defaults
+      #
+      # max_non_synchronizing_literal <em>was added in +v0.6.4+.</em>
+      #
+      # * original: +-1+ (_never_ send non-synchronizing literals)
+      # * +0.6+: 16 KiB
+      attr_accessor :max_non_synchronizing_literal, type: Integer?, defaults: {
+        0.0r => -1,
+        0.6r => 16 << 16, # 16 KiB
+      }
+
       # The maximum allowed server response size.  When +nil+, there is no limit
       # on response size.
       #

test/net/imap/fake_server.rb

@@ -103,6 +103,10 @@ def state; connection.state end
   # See CommandRouter#on
   def on(...) connection&.on(...) end
 
+  # See CommandRouter#literal_acceptor
+  def literal_acceptor = connection&.literal_acceptor
+  def literal_acceptor=(v); connection.literal_acceptor = v end
+
   # See Connection#unsolicited
   def unsolicited(...) @mutex.synchronize { connection&.unsolicited(...) } end
 

test/net/imap/fake_server/command_reader.rb

@@ -7,10 +7,12 @@ class Net::IMAP::FakeServer
 
   class CommandReader
     attr_reader :last_command
+    attr_accessor :literal_acceptor
 
     def initialize(socket)
       @socket = socket
       @last_command = nil
+      @literal_acceptor = proc {|buff, size| true }
     end
 
     def get_command
@@ -19,8 +21,17 @@ def get_command
         s = socket.gets("\r\n") or break
         buf << s
         break unless /\{(\d+)(\+)?\}\r\n\z/n =~ buf
-        $2 or socket.print "+ Continue\r\n"
-        buf << socket.read(Integer($1))
+        bytes = Integer($1)
+        if $2
+          buf << socket.read(bytes)
+        elsif literal_acceptor[buf, bytes]
+          socket.print "+ Continue\r\n"
+          buf << socket.read(bytes)
+        else
+          partial = partial_parse(buf)
+          socket.print "#{partial.tag} NO #{bytes} byte literal rejected\r\n"
+          buf = "".b
+        end
       end
       throw :eof if buf.empty?
       @last_command = parse(buf)
@@ -43,6 +54,7 @@ def parse(buf)
         Command.new $1, $2, $3, buf # TODO...
       end
     end
+    alias partial_parse parse
 
     # TODO: this is not the correct regexp, and literals aren't handled either
     def scan_astrings(str)

test/net/imap/fake_server/connection.rb

@@ -22,6 +22,9 @@ def commands; state.commands end
     def on(...) router.on(...) end
     def unsolicited(...) writer.untagged(...) end
 
+    def literal_acceptor =   reader.literal_acceptor
+    def literal_acceptor=(v) reader.literal_acceptor = v end
+
     def run
       writer.greeting
       catch(:eof) do

test/net/imap/test_command_data.rb

@@ -9,7 +9,7 @@ class CommandDataTest < Net::IMAP::TestCase
   Literal = Net::IMAP::Literal
   Literal8 = Net::IMAP::Literal8
 
-  Output = Data.define(:name, :args)
+  Output = Data.define(:name, :args, :kwargs)
   TAG = Module.new.freeze
 
   class FakeCommandWriter
@@ -18,11 +18,15 @@ def self.def_printer(name)
           Net::IMAP.private_instance_methods.include?(name)
         raise NoMethodError, "#{name} is not a method on Net::IMAP"
       end
-      define_method(name) do |*args|
-        output << Output[name:, args:]
+      define_method(name) do |*args, **kwargs|
+        kwargs = kwargs.compact
+        kwargs = nil if kwargs.empty?
+        output << Output[name:, args:, kwargs:]
       end
-      Output.define_singleton_method(name) do |*args|
-        new(name:, args:)
+      Output.define_singleton_method(name) do |*args, **kwargs|
+        kwargs = kwargs.compact
+        kwargs = nil if kwargs.empty?
+        new(name:, args:, kwargs:)
       end
     end
 
@@ -50,11 +54,20 @@ def send_data(*data, tag: TAG)
     def_printer :send_binary_literal
   end
 
+  attr_reader :imap
+
+  setup do
+    @imap = FakeCommandWriter.new
+  end
+
   test "Literal" do
-    imap = FakeCommandWriter.new
     imap.send_data Literal["foo\r\nbar"]
+    imap.send_data Literal["foo\r\nbar", false]
+    imap.send_data Literal["foo\r\nbar", true]
     assert_equal [
       Output.send_literal("foo\r\nbar", TAG),
+      Output.send_literal("foo\r\nbar", TAG, non_sync: false),
+      Output.send_literal("foo\r\nbar", TAG, non_sync: true),
     ], imap.output
 
     imap.clear
@@ -65,11 +78,14 @@ def send_data(*data, tag: TAG)
   end
 
   test "Literal8" do
-    imap = FakeCommandWriter.new
     imap.send_data Literal8["foo\r\nbar"], Literal8["foo\0bar"]
+    imap.send_data Literal8["foo\0bar", false]
+    imap.send_data Literal8["foo\0bar", true]
     assert_equal [
       Output.send_binary_literal("foo\r\nbar", TAG),
       Output.send_binary_literal("foo\0bar", TAG),
+      Output.send_binary_literal("foo\0bar", TAG, non_sync: false),
+      Output.send_binary_literal("foo\0bar", TAG, non_sync: true),
     ], imap.output
   end