fix(knowledge): use provider tokenizer for chunks and bound rerank indices

- documents/service.ts: replace ceil(len/4) heuristic with estimateTokenCount using the embedding model's tokenizerProvider so token counts match billing
- reranker.ts: filter Cohere rerank results to valid indices before mapping to defend against malformed responses
- utils.test.ts: add embeddingModel to kb fixture so getEmbeddingModelInfo resolves

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: waleedlatif1

apps/sim/app/api/knowledge/utils.test.ts

@@ -212,6 +212,7 @@ describe('Knowledge Utils', () => {
         id: 'kb1',
         userId: 'user1',
         workspaceId: null,
+        embeddingModel: 'text-embedding-3-small',
         chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 200 },
       })
       docRows.push({ id: 'doc1', knowledgeBaseId: 'kb1' })

apps/sim/lib/knowledge/documents/service.ts

@@ -34,6 +34,7 @@ import { env } from '@/lib/core/config/env'
 import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
 import { processDocument } from '@/lib/knowledge/documents/document-processor'
 import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
+import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models'
 import { generateEmbeddings } from '@/lib/knowledge/embeddings'
 import {
   buildUndefinedTagsError,
@@ -43,6 +44,7 @@ import {
   validateTagValue,
 } from '@/lib/knowledge/tags/utils'
 import type { ProcessedDocumentTags } from '@/lib/knowledge/types'
+import { estimateTokenCount } from '@/lib/tokenization/estimators'
 import { deleteFile } from '@/lib/uploads/core/storage-service'
 import { extractStorageKey } from '@/lib/uploads/utils/file-utils'
 import type { DocumentProcessingPayload } from '@/background/knowledge-processing'
@@ -533,6 +535,8 @@ export async function processDocumentAsync(
 
         logger.info(`[${documentId}] Creating embedding records with tags`)
 
+        const tokenizerProvider = getEmbeddingModelInfo(kbEmbeddingModel).tokenizerProvider
+
         const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({
           id: generateId(),
           knowledgeBaseId,
@@ -541,7 +545,7 @@ export async function processDocumentAsync(
           chunkHash: sha256Hex(chunk.text),
           content: chunk.text,
           contentLength: chunk.text.length,
-          tokenCount: Math.ceil(chunk.text.length / 4),
+          tokenCount: estimateTokenCount(chunk.text, tokenizerProvider),
           embedding: embeddings[chunkIndex] || null,
           embeddingModel: kbEmbeddingModel,
           startOffset: chunk.metadata.startIndex,

apps/sim/lib/knowledge/reranker.ts

@@ -150,10 +150,12 @@ export async function rerank<T extends RerankItem>(
   )
 
   return {
-    results: response.results.map((r) => ({
-      item: cappedItems[r.index],
-      relevanceScore: r.relevance_score,
-    })),
+    results: response.results
+      .filter((r) => r.index >= 0 && r.index < cappedItems.length)
+      .map((r) => ({
+        item: cappedItems[r.index],
+        relevanceScore: r.relevance_score,
+      })),
     isBYOK,
   }
 }