chore(chunkers): lint formatting

Author: waleedlatif1

apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx

@@ -74,9 +74,7 @@ const FormSchema = z
       .min(0, 'Overlap must be non-negative')
       .max(500, 'Overlap must be less than 500 tokens'),
     /** Chunking strategy */
-    strategy: z
-      .enum(['auto', 'text', 'regex', 'recursive', 'sentence', 'token'])
-      .default('auto'),
+    strategy: z.enum(['auto', 'text', 'regex', 'recursive', 'sentence', 'token']).default('auto'),
     /** Regex pattern (required when strategy is 'regex') */
     regexPattern: z.string().optional(),
     /** Custom separators for recursive strategy (comma-separated) */
@@ -474,14 +472,17 @@ export const CreateBaseModal = memo(function CreateBaseModal({
                       <Button
                         type='button'
                         variant='default'
-                        className='w-full justify-between border border-[var(--border-1)] !bg-[var(--surface-1)] font-normal'
+                        className='!bg-[var(--surface-1)] w-full justify-between border border-[var(--border-1)] font-normal'
                       >
                         {STRATEGY_OPTIONS.find((o) => o.value === strategyValue)?.label ??
                           'Auto (detect from content)'}
                         <ChevronDown className='h-[12px] w-[12px] text-[var(--text-icon)]' />
                       </Button>
                     </DropdownMenuTrigger>
-                    <DropdownMenuContent align='start' className='w-[var(--radix-dropdown-menu-trigger-width)]'>
+                    <DropdownMenuContent
+                      align='start'
+                      className='w-[var(--radix-dropdown-menu-trigger-width)]'
+                    >
                       <DropdownMenuRadioGroup
                         value={strategyValue}
                         onValueChange={(value) =>

apps/sim/lib/chunkers/docs-chunker.ts

@@ -218,7 +218,9 @@ export class DocsChunker {
    * Returns both the chunks and the cleaned content so header extraction
    * operates on the same text that was chunked (aligned positions).
    */
-  private async splitContent(content: string): Promise<{ chunks: string[]; cleanedContent: string }> {
+  private async splitContent(
+    content: string
+  ): Promise<{ chunks: string[]; cleanedContent: string }> {
     const cleanedContent = this.cleanContent(content)
 
     const tableBoundaries = this.detectTableBoundaries(cleanedContent)
@@ -240,19 +242,17 @@ export class DocsChunker {
    * Clean content by removing MDX-specific elements and excessive whitespace
    */
   private cleanContent(content: string): string {
-    return (
-      content
-        .replace(/\r\n/g, '\n')
-        .replace(/\r/g, '\n')
-        .replace(/^import\s+.*$/gm, '')
-        .replace(/^export\s+.*$/gm, '')
-        .replace(/<\/?[a-zA-Z][^>]*>/g, ' ')
-        .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ')
-        .replace(/\{[^{}]*\}/g, ' ')
-        .replace(/\n{3,}/g, '\n\n')
-        .replace(/[ \t]{2,}/g, ' ')
-        .trim()
-    )
+    return content
+      .replace(/\r\n/g, '\n')
+      .replace(/\r/g, '\n')
+      .replace(/^import\s+.*$/gm, '')
+      .replace(/^export\s+.*$/gm, '')
+      .replace(/<\/?[a-zA-Z][^>]*>/g, ' ')
+      .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ')
+      .replace(/\{[^{}]*\}/g, ' ')
+      .replace(/\n{3,}/g, '\n\n')
+      .replace(/[ \t]{2,}/g, ' ')
+      .trim()
   }
 
   /**

apps/sim/lib/chunkers/json-yaml-chunker.ts

@@ -56,9 +56,7 @@ export class JsonYamlChunker {
       const chunks = this.chunkStructuredData(data, [], 0)
 
       const totalTokens = chunks.reduce((sum, c) => sum + c.tokenCount, 0)
-      logger.info(
-        `JSON chunking complete: ${chunks.length} chunks, ${totalTokens} total tokens`
-      )
+      logger.info(`JSON chunking complete: ${chunks.length} chunks, ${totalTokens} total tokens`)
 
       return chunks
     } catch (error) {
@@ -84,11 +82,13 @@ export class JsonYamlChunker {
       return []
     }
 
-    return [{
-      text: content,
-      tokenCount: estimateTokens(content),
-      metadata: { startIndex: 0, endIndex: content.length },
-    }]
+    return [
+      {
+        text: content,
+        tokenCount: estimateTokens(content),
+        metadata: { startIndex: 0, endIndex: content.length },
+      },
+    ]
   }
 
   /**
@@ -108,7 +108,9 @@ export class JsonYamlChunker {
 
       if (itemTokens > this.chunkSize) {
         if (currentBatch.length > 0) {
-          chunks.push(this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1))
+          chunks.push(
+            this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1)
+          )
           currentBatch = []
           currentTokens = 0
         }
@@ -123,7 +125,9 @@ export class JsonYamlChunker {
           })
         }
       } else if (currentTokens + itemTokens > this.chunkSize && currentBatch.length > 0) {
-        chunks.push(this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1))
+        chunks.push(
+          this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1)
+        )
         currentBatch = [item]
         currentTokens = itemTokens
       } else {
@@ -133,7 +137,14 @@ export class JsonYamlChunker {
     }
 
     if (currentBatch.length > 0) {
-      chunks.push(this.buildBatchChunk(contextHeader, currentBatch, arr.length - currentBatch.length, arr.length - 1))
+      chunks.push(
+        this.buildBatchChunk(
+          contextHeader,
+          currentBatch,
+          arr.length - currentBatch.length,
+          arr.length - 1
+        )
+      )
     }
 
     return chunks
@@ -152,11 +163,13 @@ export class JsonYamlChunker {
     if (fullTokens <= this.chunkSize) {
       const contextHeader = path.length > 0 ? `// ${path.join('.')}\n` : ''
       const text = contextHeader + fullContent
-      return [{
-        text,
-        tokenCount: estimateTokens(text),
-        metadata: { startIndex: 0, endIndex: text.length },
-      }]
+      return [
+        {
+          text,
+          tokenCount: estimateTokens(text),
+          metadata: { startIndex: 0, endIndex: text.length },
+        },
+      ]
     }
 
     const contextHeader = path.length > 0 ? `// ${path.join('.')}\n` : ''
@@ -188,7 +201,10 @@ export class JsonYamlChunker {
             metadata: { startIndex: 0, endIndex: valueStr.length },
           })
         }
-      } else if (currentTokens + valueTokens > this.chunkSize && Object.keys(currentObj).length > 0) {
+      } else if (
+        currentTokens + valueTokens > this.chunkSize &&
+        Object.keys(currentObj).length > 0
+      ) {
         const objContent = contextHeader + JSON.stringify(currentObj, null, 2)
         chunks.push({
           text: objContent,
@@ -218,7 +234,12 @@ export class JsonYamlChunker {
   /**
    * Build a chunk from a batch of array items
    */
-  private buildBatchChunk(contextHeader: string, batch: JsonValue[], startIdx: number, endIdx: number): Chunk {
+  private buildBatchChunk(
+    contextHeader: string,
+    batch: JsonValue[],
+    startIdx: number,
+    endIdx: number
+  ): Chunk {
     const batchContent = contextHeader + JSON.stringify(batch, null, 2)
     return {
       text: batchContent,

apps/sim/lib/chunkers/sentence-chunker.ts

@@ -34,7 +34,9 @@ export class SentenceChunker {
    */
   private splitSentences(text: string): string[] {
     return text
-      .split(/(?<!\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr|St|Rev|Gen|Sgt|No|Fig|Vol|Ch|vs|etc|Inc|Ltd|Corp|approx|dept|est|govt|Jan|Feb|Mar|Apr|Aug|Sep|Oct|Nov|Dec|i\.e|e\.g))(?<![A-Z])(?<!\.\.)(?<!\d)(?<=[.!?])\s+/)
+      .split(
+        /(?<!\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr|St|Rev|Gen|Sgt|No|Fig|Vol|Ch|vs|etc|Inc|Ltd|Corp|approx|dept|est|govt|Jan|Feb|Mar|Apr|Aug|Sep|Oct|Nov|Dec|i\.e|e\.g))(?<![A-Z])(?<!\.\.)(?<!\d)(?<=[.!?])\s+/
+      )
       .filter((s) => s.trim().length > 0)
   }
 

apps/sim/lib/chunkers/structured-data-chunker.ts

@@ -179,9 +179,7 @@ export class StructuredDataChunker {
     const delimiters = [',', '\t', '|']
     for (const delimiter of delimiters) {
       const escaped = delimiter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-      const counts = lines.map(
-        (line) => (line.match(new RegExp(escaped, 'g')) || []).length
-      )
+      const counts = lines.map((line) => (line.match(new RegExp(escaped, 'g')) || []).length)
       const avgCount = counts.reduce((a, b) => a + b, 0) / counts.length
 
       const tolerance = Math.max(1, Math.ceil(avgCount * 0.2))

apps/sim/lib/chunkers/utils.ts

@@ -106,10 +106,7 @@ export function splitAtWordBoundaries(
 /**
  * Build Chunk objects from text segments with startIndex/endIndex metadata
  */
-export function buildChunks(
-  texts: string[],
-  overlapTokens: number
-): Chunk[] {
+export function buildChunks(texts: string[], overlapTokens: number): Chunk[] {
   let previousEndIndex = 0
   const overlapChars = tokensToChars(overlapTokens)
 

apps/sim/lib/knowledge/documents/document-processor.ts

@@ -154,7 +154,9 @@ async function applyStrategy(
     }
     case 'regex': {
       if (!strategyOptions?.pattern) {
-        logger.warn('Regex strategy requested but no pattern provided, falling back to text chunker')
+        logger.warn(
+          'Regex strategy requested but no pattern provided, falling back to text chunker'
+        )
         const chunker = new TextChunker(baseOptions)
         return chunker.chunk(content)
       }
@@ -164,7 +166,6 @@ async function applyStrategy(
       })
       return chunker.chunk(content)
     }
-    case 'text':
     default: {
       const chunker = new TextChunker(baseOptions)
       return chunker.chunk(content)

apps/sim/lib/knowledge/documents/service.ts

@@ -27,12 +27,12 @@ import {
 } from 'drizzle-orm'
 import { recordUsage } from '@/lib/billing/core/usage-log'
 import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing'
+import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
 import { createBullMQJobData, isBullMQEnabled } from '@/lib/core/bullmq'
 import { env } from '@/lib/core/config/env'
 import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
 import { generateId } from '@/lib/core/utils/uuid'
 import { enqueueWorkspaceDispatch } from '@/lib/core/workspace-dispatch'
-import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
 import { processDocument } from '@/lib/knowledge/documents/document-processor'
 import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
 import { generateEmbeddings } from '@/lib/knowledge/embeddings'