Integrate with Sentence Transformers via MultiVectorEncoder

1_Dense/config.json

@@ -0,0 +1,8 @@
+{
+  "in_features": 2560,
+  "out_features": 2048,
+  "bias": true,
+  "activation_function": "torch.nn.modules.linear.Identity",
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

1_Dense/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0af96e01abcb912f81c4bd5cee1acb4460e0b97184b59f337be63cd15fc68631
+size 20979880

2_Normalize/config.json

@@ -0,0 +1,4 @@
+{
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

3_MultiVectorMask/config.json

@@ -0,0 +1,3 @@
+{
+  "skiplist_words": []
+}

README.md

@@ -13,6 +13,7 @@ tags:
 - matryoshka
 - vidore
 - token-compression
+- sentence-transformers
 datasets:
 - vidore/vidore_benchmark
 - vidore/vidore_benchmark_v2
@@ -191,7 +192,9 @@ Protocol `paired-all-pages-dedup+process_queries+ndcg2r-20260827` ($\text{MVT} =
 
 ## ⚡ Quick Start
 
-### Installation
+### ColPali Engine
+
+#### Installation
 
 ```bash
 git clone https://github.com/Tencent/EVIE.git
@@ -200,7 +203,7 @@ pip install -r requirements.txt
 export PYTHONPATH="$(pwd)/colpali${PYTHONPATH:+:$PYTHONPATH}"
 ```
 
-### Self-Contained Python Inference
+#### Self-Contained Python Inference
 
 ```python
 import torch
@@ -240,6 +243,54 @@ scores = processor.score(query_embeddings, image_embeddings)
 print("Late-interaction MaxSim Relevance Score:", scores)
 ```
 
+### Sentence Transformers
+
+Load EVIE with [Sentence Transformers](https://www.sbert.net/) to encode queries and document images and compute MaxSim scores. Bidirectional attention and the 768-token visual budget are configured automatically.
+
+```bash
+pip install -U "sentence-transformers[image]>=6.0.0"
+```
+
+```python
+from sentence_transformers import MultiVectorEncoder
+
+model = MultiVectorEncoder("tencent/EVIE-4.5B")
+
+queries = [
+    "What is the variable represented on the y-axis of the graph?",
+    "Total outlay is maximum in which year?",
+]
+documents = [
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc1.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc2.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc3.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc4.jpg",
+]
+
+query_embeddings = model.encode_query(queries)
+document_embeddings = model.encode_document(documents)
+print(query_embeddings[0].shape, document_embeddings[0].shape)
+# torch.Size([23, 2048]) torch.Size([755, 2048])
+
+scores = model.similarity(query_embeddings, document_embeddings)
+print(scores)
+# tensor([[17.0977,  7.8730,  6.9902,  4.8066],
+#         [ 4.4795, 12.8516,  4.2031,  4.3779]])
+```
+
+Documents can be URLs, local image paths, or `PIL.Image` objects. The example returns the full 2048-dimensional token embeddings. Scores can vary slightly with dtype and attention backend.
+
+To use a smaller Matryoshka prefix, slice and renormalize each token embedding before scoring. Supported dimensions are 64, 128, 256, 512, 1024, and 2048:
+
+```python
+from torch.nn.functional import normalize
+
+dimension = 128
+query_embeddings = [normalize(embedding[..., :dimension], dim=-1) for embedding in query_embeddings]
+document_embeddings = [normalize(embedding[..., :dimension], dim=-1) for embedding in document_embeddings]
+scores = model.similarity(query_embeddings, document_embeddings)
+```
+
 ---
 
 ## 📂 Repository Layout

additional_chat_templates/sentence_transformers.jinja

@@ -0,0 +1,19 @@
+{%- for message in messages -%}
+{%- set ns = namespace(has_image=false, text='') -%}
+{%- if message['content'] is string -%}
+{%- set ns.text = message['content'] -%}
+{%- else -%}
+{%- for item in message['content'] -%}
+{%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}
+{%- set ns.has_image = true -%}
+{%- elif 'text' in item -%}
+{%- set ns.text = ns.text + item.text -%}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
+{%- if ns.has_image -%}
+{{- '<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|><|endoftext|>' -}}
+{%- else -%}
+{{- ns.text + '<|endoftext|>' * 10 -}}
+{%- endif -%}
+{%- endfor -%}

config_sentence_transformers.json

@@ -0,0 +1,9 @@
+{
+  "__version__": {
+    "sentence_transformers": "6.0.0"
+  },
+  "model_type": "MultiVectorEncoder",
+  "similarity_fn_name": "maxsim",
+  "prompts": {},
+  "default_prompt_name": null
+}

modules.json

@@ -0,0 +1,26 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.base.modules.transformer.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Dense",
+    "type": "sentence_transformers.base.modules.dense.Dense"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.sentence_transformer.modules.normalize.Normalize"
+  },
+  {
+    "idx": 3,
+    "name": "3",
+    "path": "3_MultiVectorMask",
+    "type": "sentence_transformers.multi_vector_encoder.modules.multi_vector_mask.MultiVectorMask"
+  }
+]

processor_config.json

@@ -25,7 +25,7 @@
     },
     "temporal_patch_size": 2
   },
-  "processor_class": "ColQwen3_5Processor",
+  "processor_class": "Qwen3VLProcessor",
   "video_processor": {
     "do_convert_rgb": true,
     "do_normalize": true,

sentence_bert_config.json

@@ -0,0 +1,30 @@
+{
+  "transformer_task": "feature-extraction",
+  "modality_config": {
+    "text": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "image": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "message": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state",
+      "format": "structured"
+    }
+  },
+  "module_output_name": "token_embeddings",
+  "unpad_inputs": false,
+  "config_kwargs": {
+    "text_config": {
+      "is_causal": false
+    }
+  },
+  "processing_kwargs": {
+    "chat_template": {
+      "chat_template": "sentence_transformers"
+    }
+  }
+}

tokenizer_config.json

@@ -24,7 +24,8 @@
   },
   "pad_token": "<|endoftext|>",
   "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
-  "processor_class": "ColQwen3_5Processor",
+  "processor_class": "Qwen3VLProcessor",
+  "padding_side": "left",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null,