Merge remote-tracking branch 'origin/master' into feat/wavlm

Author: a2d8a4v

.github/workflows/ci.yml

@@ -167,8 +167,8 @@ jobs:
           CIBW_BEFORE_ALL_MACOS: python/tools/prepare_build_environment_macos.sh
           CIBW_BEFORE_ALL_WINDOWS: bash python/tools/prepare_build_environment_windows.sh
           CIBW_BEFORE_BUILD: pip install -r python/install_requirements.txt
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
           CIBW_ARCHS: ${{ matrix.arch }}
           CIBW_SKIP: "*-musllinux_*"
 

CHANGELOG.md

@@ -4,6 +4,20 @@
 
 ### Fixes and improvements
 
+## [v4.6.3](https://github.com/OpenNMT/CTranslate2/releases/tag/v4.6.3) (2026-01-XX)
+
+### New features
+
+* T5Gemma model conversion and inference (#1962) by [@jordimas](https://github.com/jordimas)
+* Support for CUDA 12.8 (#1937, #1940) by [@Purfview](https://github.com/Purfview)
+* Conv1d pure CUDA implementation (#1949), makes cuDNN an optional dependency by [@jordimas](https://github.com/jordimas)
+* Add CUDA implementation for median filter (#1917) by [@ja2d8a4v](https://github.com/a2d8a4v)
+
+### Fixes and improvements
+
+* Fix issue- tokens batch_type may exceed max_batch_size (#1948) by [@anterart](https://github.com/anterart)
+* Fixes AWQ support for Qwen 2.5/3 and Phi3 (#1951, #1952, #1953) by [@BBC-Esq](https://github.com/BBC-Esq)
+
 ## [v4.6.2](https://github.com/OpenNMT/CTranslate2/releases/tag/v4.6.2) (2025-12-05)
 
 ### New features
@@ -29,7 +43,7 @@
 Note: The Ctranslate2 Python package now supports python 3.13, drop the support for python 3.8.
 
 ### New features
-* Pyhton 3.13 support (#1858)
+* Python 3.13 support (#1858)
 * Support returning hidden vector in Wav2Vec2 and Wav2Vec2Bert Models (#1867)
 * Add noexecstack linker flags (#1852 + #1861)
 * Support Qwen2 (#1820)

README.md

@@ -8,7 +8,7 @@ The project implements a custom runtime that applies many performance optimizati
 
 The following model types are currently supported:
 
-* Encoder-decoder models: Transformer base/big, M2M-100, NLLB, BART, mBART, Pegasus, T5, Whisper
+* Encoder-decoder models: Transformer base/big, M2M-100, NLLB, BART, mBART, Pegasus, T5, Whisper T5Gemma
 * Decoder-only models: GPT-2, GPT-J, GPT-NeoX, OPT, BLOOM, MPT, Llama, Mistral, Gemma, CodeGen, GPTBigCode, Falcon, Qwen2
 * Encoder-only models: BERT, DistilBERT, XLM-RoBERTa
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 as builder
+FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 AS builder
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -77,21 +77,21 @@ RUN cd python && \
     python3 -m pip --no-cache-dir install -r install_requirements.txt && \
     python3 setup.py bdist_wheel --dist-dir $CTRANSLATE2_ROOT
 
-FROM nvidia/cuda:12.4.1-base-ubuntu22.04
+FROM nvidia/cuda:12.8.1-base-ubuntu22.04
 
 # We remove the cuda-compat package because it conflicts with the CUDA Enhanced Compatibility.
 # See e.g. https://github.com/NVIDIA/nvidia-docker/issues/1515
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        libcublas-12-4 \
-        libcudnn9-cuda-12 \
-        libnccl2 \
+        libcublas-12-8=12.8.4.1-1 \
+        libcudnn9-cuda-12=9.10.2.21-1 \
+        libnccl2=2.26.2-1+cuda12.8 \
         libopenmpi3=4.1.2-2ubuntu1 \
         openmpi-bin \
         libgomp1 \
         python3-pip \
         && \
-    apt-get purge -y cuda-compat-12-4 && \
+    apt-get purge -y cuda-compat-12-8 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

docs/guides/transformers.md

@@ -25,6 +25,7 @@ CTranslate2 supports selected models from Hugging Face's [Transformers](https://
 * Qwen 2.5
 * Qwen 3
 * T5
+* T5Gemma
 * Whisper
 * XLM-RoBERTa
 
@@ -557,6 +558,46 @@ output_text = tokenizer.decode(tokenizer.convert_tokens_to_ids(output_tokens))
 print(output_text)
 ```
 
+## T5Gemma
+
+[T5Gemma](https://deepmind.google/models/gemma/t5gemma/) is collection of Google encoder-decoder models that provide a strong quality-inference efficiency tradeoff.
+
+To convert a model:
+
+```bash
+ct2-transformers-converter --model google/t5gemma-b-b-prefixlm-it --output_dir t5gemma_b_b_prefixlm_it.ct2
+```
+
+Usage:
+
+```python
+import ctranslate2
+import transformers
+
+translator = ctranslate2.Translator("t5gemma_b_b_prefixlm_it.ct2")
+tokenizer = transformers.AutoTokenizer.from_pretrained("google/t5gemma-b-b-prefixlm-it")
+
+sentences = ["Question: Why is the sky blue? Answer:"]
+
+# Tokenize each sentence
+tokenized_sentences = [
+    tokenizer.convert_ids_to_tokens(tokenizer.encode(sentence))
+    for sentence in sentences
+]
+
+translated_batches = translator.translate_batch(tokenized_sentences, beam_size=1, repetition_penalty=1.2, max_decoding_length=50)
+
+
+# Decode results
+translations = [
+    tokenizer.decode(tokenizer.convert_tokens_to_ids(t.hypotheses[0]))
+    for t in translated_batches
+]
+final_translation = " ".join(translations)
+print(final_translation)
+```
+
+
 ## Whisper
 
 [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper) is a multilingual speech recognition model published by OpenAI.

include/ctranslate2/devices.h

@@ -3,6 +3,7 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
+#include <cstdint>
 #ifdef CT2_WITH_TENSOR_PARALLEL
 #  include <nccl.h>
 #endif

include/ctranslate2/layers/attention.h

@@ -47,6 +47,20 @@ namespace ctranslate2 {
       virtual bool has_positional_embeddings() const override {
             return _relative_position_keys || _relative_attention_bias || _rotary_embeddings || _alibi;
       }
+
+    protected:
+      void process_cross_attention(const StorageView& queries,
+                                const StorageView& values,
+                                StorageView& fused_proj,
+                                StorageView& queries_proj,
+                                StorageView& keys_proj,
+                                StorageView& values_proj,
+                                StorageView* cached_keys,
+                                StorageView* cached_values,
+                                const Padder* queries_padder,
+                                const Padder* values_padder,
+                                dim_t& beam_size) const;
+
     private:
       static void split_heads(StorageView& x,
                                dim_t num_heads,
@@ -57,6 +71,10 @@ namespace ctranslate2 {
                                  dim_t num_heads,
                                  const Padder* padder = nullptr,
                                  dim_t beam_size = 1);
+
+      void apply_qk_norm(StorageView& queries_proj,
+                          StorageView& keys_proj) const;
+
       const StorageView* _relative_attention_bias;
       const StorageView* _relative_position_keys;
       const StorageView* _relative_asymmetric_position_keys;

include/ctranslate2/layers/transformer.h

@@ -68,6 +68,10 @@ namespace ctranslate2 {
 
     private:
       std::unique_ptr<AttentionLayer> _self_attention;
+      const std::unique_ptr<const LayerNorm> _input_layer_norm;
+      const std::unique_ptr<const LayerNorm> _post_attention_layer_norm;
+      const std::unique_ptr<const LayerNorm> _pre_feedforward_layer_norm;
+      const std::unique_ptr<const LayerNorm> _post_feedforward_layer_norm;
       const FeedForwardNetwork _ff;
     };
 
@@ -123,6 +127,8 @@ namespace ctranslate2 {
       const std::unique_ptr<const LayerNorm> _post_feedforward_layer_norm;
       const std::unique_ptr<const AttentionLayer> _encoder_attention;
       const FeedForwardNetwork _ff;
+      const std::unique_ptr<const LayerNorm> _external_pre_encoder_attention_layer_norm;
+      const std::unique_ptr<const LayerNorm> _external_post_encoder_attention_layer_norm;
     };
 
     class TransformerEncoder : public Encoder

include/ctranslate2/layers/wav2vec2.h

@@ -27,8 +27,8 @@ namespace ctranslate2 {
       dim_t _stride;
       dim_t _padding;
       const Conv1D _conv;
-      const LayerNorm _output_norm;
       const ops::Transpose _transpose;
+      const LayerNorm _output_norm;
       const ops::GELU _gelu;
     };
 

include/ctranslate2/models/model.h

@@ -171,10 +171,12 @@ namespace ctranslate2 {
       virtual void initialize(ModelReader&) {}
 
       virtual std::unique_ptr<Model> clone() const = 0;
+      
+      void set_compute_type(ComputeType type, Device device, int device_index, bool update_weight=true);
+
 
     private:
       void process_linear_weights();
-      void set_compute_type(ComputeType type, Device device, int device_index, bool update_weight=true);
       void ensure_dtype(const std::string& name,
                         StorageView& variable,
                         const DataType target_dtype);