feat(tiktoken): improve token counting accuracy (#1382)

Use more accurate token prediction when tiktoken core is not available

Signed-off-by: Tomas Slusny <slusnucky@gmail.com>

Author: deathbeam

lua/CopilotChat/client.lua

@@ -362,6 +362,10 @@ function Client:ask(prompt, opts)
     local resource_tokens = #resource_messages > 0 and tiktoken:count(resource_messages[1].content) or 0
     local required_tokens = prompt_tokens + system_tokens + resource_tokens
 
+    log.debug('Prompt tokens:', prompt_tokens)
+    log.debug('System tokens:', system_tokens)
+    log.debug('Resource tokens:', resource_tokens)
+
     -- Calculate how many tokens we can use for history
     local history_limit = max_tokens - required_tokens
     local history_tokens = 0

lua/CopilotChat/tiktoken.lua

@@ -1,3 +1,4 @@
+local log = require('plenary.log')
 local notify = require('CopilotChat.notify')
 local utils = require('CopilotChat.utils')
 local curl = require('CopilotChat.utils.curl')
@@ -105,12 +106,12 @@ end
 ---@return number
 function Tiktoken:count(prompt)
   if not self.tiktoken_core then
-    return math.ceil(#prompt * 0.5) -- Fallback to 1/2 character count
+    return math.ceil(#prompt / 4)
   end
 
   local tokens = self:encode(prompt)
   if not tokens then
-    return math.ceil(#prompt * 0.5) -- Fallback to 1/2 character count
+    return math.ceil(#prompt / 4)
   end
   return #tokens
 end