feat: add full support for number/date/boolean tag filtering in KB search

- Copy all tag types (number, date, boolean) from document to embedding records
- Update processDocumentTags to handle all field types with proper type conversion
- Add number/date/boolean columns to document queries in checkDocumentWriteAccess
- Update chunk creation to inherit all tag types from parent document
- Add getSearchResultFields helper for consistent query result selection
- Support structured filters with operators (eq, gt, lt, between, etc.)
- Fix search queries to include all 28 tag fields in results

Author: priyanshu.solanki

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts

@@ -156,13 +156,38 @@ export async function POST(
       const validatedData = CreateChunkSchema.parse(searchParams)
 
       const docTags = {
+        // Text tags
         tag1: doc.tag1 ?? null,
         tag2: doc.tag2 ?? null,
         tag3: doc.tag3 ?? null,
         tag4: doc.tag4 ?? null,
         tag5: doc.tag5 ?? null,
         tag6: doc.tag6 ?? null,
         tag7: doc.tag7 ?? null,
+        // Number tags
+        number1: doc.number1 ?? null,
+        number2: doc.number2 ?? null,
+        number3: doc.number3 ?? null,
+        number4: doc.number4 ?? null,
+        number5: doc.number5 ?? null,
+        number6: doc.number6 ?? null,
+        number7: doc.number7 ?? null,
+        // Date tags
+        date1: doc.date1 ?? null,
+        date2: doc.date2 ?? null,
+        date3: doc.date3 ?? null,
+        date4: doc.date4 ?? null,
+        date5: doc.date5 ?? null,
+        date6: doc.date6 ?? null,
+        date7: doc.date7 ?? null,
+        // Boolean tags
+        boolean1: doc.boolean1 ?? null,
+        boolean2: doc.boolean2 ?? null,
+        boolean3: doc.boolean3 ?? null,
+        boolean4: doc.boolean4 ?? null,
+        boolean5: doc.boolean5 ?? null,
+        boolean6: doc.boolean6 ?? null,
+        boolean7: doc.boolean7 ?? null,
       }
 
       const newChunk = await createChunk(

apps/sim/app/api/knowledge/search/route.ts

@@ -20,6 +20,16 @@ import { calculateCost } from '@/providers/utils'
 
 const logger = createLogger('VectorSearchAPI')
 
+/** Structured tag filter with operator support */
+const StructuredTagFilterSchema = z.object({
+  tagName: z.string(),
+  tagSlot: z.string().optional(),
+  fieldType: z.enum(['text', 'number', 'date', 'boolean']).default('text'),
+  operator: z.string().default('eq'),
+  value: z.union([z.string(), z.number(), z.boolean()]),
+  valueTo: z.union([z.string(), z.number()]).optional(),
+})
+
 const VectorSearchSchema = z
   .object({
     knowledgeBaseIds: z.union([
@@ -43,14 +53,20 @@ const VectorSearchSchema = z
       .record(z.string())
       .optional()
       .nullable()
-      .transform((val) => val || undefined), // Allow dynamic filter keys (display names)
+      .transform((val) => val || undefined), // Legacy format: simple key-value pairs
+    tagFilters: z
+      .array(StructuredTagFilterSchema)
+      .optional()
+      .nullable()
+      .transform((val) => val || undefined), // New format: structured filters with operators
   })
   .refine(
     (data) => {
       // Ensure at least query or filters are provided
       const hasQuery = data.query && data.query.trim().length > 0
-      const hasFilters = data.filters && Object.keys(data.filters).length > 0
-      return hasQuery || hasFilters
+      const hasLegacyFilters = data.filters && Object.keys(data.filters).length > 0
+      const hasTagFilters = data.tagFilters && data.tagFilters.length > 0
+      return hasQuery || hasLegacyFilters || hasTagFilters
     },
     {
       message: 'Please provide either a search query or tag filters to search your knowledge base',
@@ -89,6 +105,54 @@ export async function POST(request: NextRequest) {
 
       // Map display names to tag slots for filtering
       let mappedFilters: Record<string, string> = {}
+      let structuredFilters: Array<{
+        tagSlot: string
+        fieldType: string
+        operator: string
+        value: string | number | boolean
+        valueTo?: string | number
+      }> = []
+
+      // Handle new structured tagFilters format
+      if (validatedData.tagFilters && accessibleKbIds.length > 0) {
+        try {
+          const kbId = accessibleKbIds[0]
+          const tagDefs = await getDocumentTagDefinitions(kbId)
+
+          // Create mapping from display name to tag slot and fieldType
+          const displayNameToTagDef: Record<string, { tagSlot: string; fieldType: string }> = {}
+          tagDefs.forEach((def) => {
+            displayNameToTagDef[def.displayName] = {
+              tagSlot: def.tagSlot,
+              fieldType: def.fieldType,
+            }
+          })
+
+          structuredFilters = validatedData.tagFilters.map((filter) => {
+            const tagDef = displayNameToTagDef[filter.tagName]
+            const tagSlot = filter.tagSlot || tagDef?.tagSlot || filter.tagName
+            const fieldType = filter.fieldType || tagDef?.fieldType || 'text'
+
+            logger.debug(
+              `[${requestId}] Structured filter: ${filter.tagName} -> ${tagSlot} (${fieldType}) ${filter.operator} ${filter.value}`
+            )
+
+            return {
+              tagSlot,
+              fieldType,
+              operator: filter.operator,
+              value: filter.value,
+              valueTo: filter.valueTo,
+            }
+          })
+
+          logger.debug(`[${requestId}] Processed ${structuredFilters.length} structured filters`)
+        } catch (error) {
+          logger.error(`[${requestId}] Structured filter processing error:`, error)
+        }
+      }
+
+      // Handle legacy filters format (for backwards compatibility)
       if (validatedData.filters && accessibleKbIds.length > 0) {
         try {
           // Fetch tag definitions for the first accessible KB (since we're using single KB now)
@@ -155,26 +219,36 @@ export async function POST(request: NextRequest) {
 
       let results: SearchResult[]
 
-      const hasFilters = mappedFilters && Object.keys(mappedFilters).length > 0
+      const hasLegacyFilters = mappedFilters && Object.keys(mappedFilters).length > 0
+      const hasStructuredFilters = structuredFilters && structuredFilters.length > 0
+      const hasFilters = hasLegacyFilters || hasStructuredFilters
 
       if (!hasQuery && hasFilters) {
         // Tag-only search without vector similarity
-        logger.debug(`[${requestId}] Executing tag-only search with filters:`, mappedFilters)
+        logger.debug(
+          `[${requestId}] Executing tag-only search with filters:`,
+          hasStructuredFilters ? structuredFilters : mappedFilters
+        )
         results = await handleTagOnlySearch({
           knowledgeBaseIds: accessibleKbIds,
           topK: validatedData.topK,
-          filters: mappedFilters,
+          filters: hasLegacyFilters ? mappedFilters : undefined,
+          structuredFilters: hasStructuredFilters ? structuredFilters : undefined,
         })
       } else if (hasQuery && hasFilters) {
         // Tag + Vector search
-        logger.debug(`[${requestId}] Executing tag + vector search with filters:`, mappedFilters)
+        logger.debug(
+          `[${requestId}] Executing tag + vector search with filters:`,
+          hasStructuredFilters ? structuredFilters : mappedFilters
+        )
         const strategy = getQueryStrategy(accessibleKbIds.length, validatedData.topK)
         const queryVector = JSON.stringify(await queryEmbeddingPromise)
 
         results = await handleTagAndVectorSearch({
           knowledgeBaseIds: accessibleKbIds,
           topK: validatedData.topK,
-          filters: mappedFilters,
+          filters: hasLegacyFilters ? mappedFilters : undefined,
+          structuredFilters: hasStructuredFilters ? structuredFilters : undefined,
           queryVector,
           distanceThreshold: strategy.distanceThreshold,
         })