Skip to content

Commit 7f22628

Browse files
waleedlatif1waleedlatif1
authored
improvement(kb): add document filtering, select all, and React Query migration (#2951)
* improvement(kb): add document filtering, select all, and React Query migration * test(kb): update tests for enabledFilter and removed userId params * fix(kb): remove non-null assertion, add explicit guard
1 parent 1b309b5 commit 7f22628

File tree

27 files changed

+844
-698
lines changed

27 files changed

+844
-698
lines changed

‎apps/sim/app/api/knowledge/[id]/documents/route.test.ts‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ describe('Knowledge Base Documents API Route', () => {
157157
expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
158158
'kb-123',
159159
{
160-
includeDisabled: false,
160+
enabledFilter: undefined,
161161
search: undefined,
162162
limit: 50,
163163
offset: 0,
@@ -166,7 +166,7 @@ describe('Knowledge Base Documents API Route', () => {
166166
)
167167
})
168168

169-
it('should filter disabled documents by default', async () => {
169+
it('should return documents with default filter', async () => {
170170
const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
171171
const { getDocuments } = await import('@/lib/knowledge/documents/service')
172172

@@ -194,7 +194,7 @@ describe('Knowledge Base Documents API Route', () => {
194194
expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
195195
'kb-123',
196196
{
197-
includeDisabled: false,
197+
enabledFilter: undefined,
198198
search: undefined,
199199
limit: 50,
200200
offset: 0,
@@ -203,7 +203,7 @@ describe('Knowledge Base Documents API Route', () => {
203203
)
204204
})
205205

206-
it('should include disabled documents when requested', async () => {
206+
it('should filter documents by enabled status when requested', async () => {
207207
const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
208208
const { getDocuments } = await import('@/lib/knowledge/documents/service')
209209

@@ -223,7 +223,7 @@ describe('Knowledge Base Documents API Route', () => {
223223
},
224224
})
225225

226-
const url = 'http://localhost:3000/api/knowledge/kb-123/documents?includeDisabled=true'
226+
const url = 'http://localhost:3000/api/knowledge/kb-123/documents?enabledFilter=disabled'
227227
const req = new Request(url, { method: 'GET' }) as any
228228

229229
const { GET } = await import('@/app/api/knowledge/[id]/documents/route')
@@ -233,7 +233,7 @@ describe('Knowledge Base Documents API Route', () => {
233233
expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
234234
'kb-123',
235235
{
236-
includeDisabled: true,
236+
enabledFilter: 'disabled',
237237
search: undefined,
238238
limit: 50,
239239
offset: 0,
@@ -361,8 +361,7 @@ describe('Knowledge Base Documents API Route', () => {
361361
expect(vi.mocked(createSingleDocument)).toHaveBeenCalledWith(
362362
validDocumentData,
363363
'kb-123',
364-
expect.any(String),
365-
'user-123'
364+
expect.any(String)
366365
)
367366
})
368367

@@ -470,8 +469,7 @@ describe('Knowledge Base Documents API Route', () => {
470469
expect(vi.mocked(createDocumentRecords)).toHaveBeenCalledWith(
471470
validBulkData.documents,
472471
'kb-123',
473-
expect.any(String),
474-
'user-123'
472+
expect.any(String)
475473
)
476474
expect(vi.mocked(processDocumentsWithQueue)).toHaveBeenCalled()
477475
})

‎apps/sim/app/api/knowledge/[id]/documents/route.ts‎

Lines changed: 38 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { z } from 'zod'
55
import { getSession } from '@/lib/auth'
66
import {
77
bulkDocumentOperation,
8+
bulkDocumentOperationByFilter,
89
createDocumentRecords,
910
createSingleDocument,
1011
getDocuments,
@@ -57,13 +58,20 @@ const BulkCreateDocumentsSchema = z.object({
5758
bulk: z.literal(true),
5859
})
5960

60-
const BulkUpdateDocumentsSchema = z.object({
61-
operation: z.enum(['enable', 'disable', 'delete']),
62-
documentIds: z
63-
.array(z.string())
64-
.min(1, 'At least one document ID is required')
65-
.max(100, 'Cannot operate on more than 100 documents at once'),
66-
})
61+
const BulkUpdateDocumentsSchema = z
62+
.object({
63+
operation: z.enum(['enable', 'disable', 'delete']),
64+
documentIds: z
65+
.array(z.string())
66+
.min(1, 'At least one document ID is required')
67+
.max(100, 'Cannot operate on more than 100 documents at once')
68+
.optional(),
69+
selectAll: z.boolean().optional(),
70+
enabledFilter: z.enum(['all', 'enabled', 'disabled']).optional(),
71+
})
72+
.refine((data) => data.selectAll || (data.documentIds && data.documentIds.length > 0), {
73+
message: 'Either selectAll must be true or documentIds must be provided',
74+
})
6775

6876
export async function GET(req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
6977
const requestId = randomUUID().slice(0, 8)
@@ -90,21 +98,25 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
9098
}
9199

92100
const url = new URL(req.url)
93-
const includeDisabled = url.searchParams.get('includeDisabled') === 'true'
101+
const enabledFilter = url.searchParams.get('enabledFilter') as
102+
| 'all'
103+
| 'enabled'
104+
| 'disabled'
105+
| null
94106
const search = url.searchParams.get('search') || undefined
95107
const limit = Number.parseInt(url.searchParams.get('limit') || '50')
96108
const offset = Number.parseInt(url.searchParams.get('offset') || '0')
97109
const sortByParam = url.searchParams.get('sortBy')
98110
const sortOrderParam = url.searchParams.get('sortOrder')
99111

100-
// Validate sort parameters
101112
const validSortFields: DocumentSortField[] = [
102113
'filename',
103114
'fileSize',
104115
'tokenCount',
105116
'chunkCount',
106117
'uploadedAt',
107118
'processingStatus',
119+
'enabled',
108120
]
109121
const validSortOrders: SortOrder[] = ['asc', 'desc']
110122

@@ -120,7 +132,7 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
120132
const result = await getDocuments(
121133
knowledgeBaseId,
122134
{
123-
includeDisabled,
135+
enabledFilter: enabledFilter || undefined,
124136
search,
125137
limit,
126138
offset,
@@ -190,8 +202,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
190202
const createdDocuments = await createDocumentRecords(
191203
validatedData.documents,
192204
knowledgeBaseId,
193-
requestId,
194-
userId
205+
requestId
195206
)
196207

197208
logger.info(
@@ -250,16 +261,10 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
250261
throw validationError
251262
}
252263
} else {
253-
// Handle single document creation
254264
try {
255265
const validatedData = CreateDocumentSchema.parse(body)
256266

257-
const newDocument = await createSingleDocument(
258-
validatedData,
259-
knowledgeBaseId,
260-
requestId,
261-
userId
262-
)
267+
const newDocument = await createSingleDocument(validatedData, knowledgeBaseId, requestId)
263268

264269
try {
265270
const { PlatformEvents } = await import('@/lib/core/telemetry')
@@ -294,7 +299,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
294299
} catch (error) {
295300
logger.error(`[${requestId}] Error creating document`, error)
296301

297-
// Check if it's a storage limit error
298302
const errorMessage = error instanceof Error ? error.message : 'Failed to create document'
299303
const isStorageLimitError =
300304
errorMessage.includes('Storage limit exceeded') || errorMessage.includes('storage limit')
@@ -331,16 +335,22 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id
331335

332336
try {
333337
const validatedData = BulkUpdateDocumentsSchema.parse(body)
334-
const { operation, documentIds } = validatedData
338+
const { operation, documentIds, selectAll, enabledFilter } = validatedData
335339

336340
try {
337-
const result = await bulkDocumentOperation(
338-
knowledgeBaseId,
339-
operation,
340-
documentIds,
341-
requestId,
342-
session.user.id
343-
)
341+
let result
342+
if (selectAll) {
343+
result = await bulkDocumentOperationByFilter(
344+
knowledgeBaseId,
345+
operation,
346+
enabledFilter,
347+
requestId
348+
)
349+
} else if (documentIds && documentIds.length > 0) {
350+
result = await bulkDocumentOperation(knowledgeBaseId, operation, documentIds, requestId)
351+
} else {
352+
return NextResponse.json({ error: 'No documents specified' }, { status: 400 })
353+
}
344354

345355
return NextResponse.json({
346356
success: true,

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/edit-chunk-modal/edit-chunk-modal.tsx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ export function EditChunkModal({
6161
const [showUnsavedChangesAlert, setShowUnsavedChangesAlert] = useState(false)
6262
const [pendingNavigation, setPendingNavigation] = useState<(() => void) | null>(null)
6363
const [tokenizerOn, setTokenizerOn] = useState(false)
64+
const [hoveredTokenIndex, setHoveredTokenIndex] = useState<number | null>(null)
6465
const textareaRef = useRef<HTMLTextAreaElement>(null)
6566

6667
const error = mutationError?.message ?? null
@@ -254,6 +255,8 @@ export function EditChunkModal({
254255
style={{
255256
backgroundColor: getTokenBgColor(index),
256257
}}
258+
onMouseEnter={() => setHoveredTokenIndex(index)}
259+
onMouseLeave={() => setHoveredTokenIndex(null)}
257260
>
258261
{token}
259262
</span>
@@ -281,6 +284,11 @@ export function EditChunkModal({
281284
<div className='flex items-center gap-[8px]'>
282285
<span className='text-[12px] text-[var(--text-secondary)]'>Tokenizer</span>
283286
<Switch checked={tokenizerOn} onCheckedChange={setTokenizerOn} />
287+
{tokenizerOn && hoveredTokenIndex !== null && (
288+
<span className='text-[12px] text-[var(--text-tertiary)]'>
289+
Token #{hoveredTokenIndex + 1}
290+
</span>
291+
)}
284292
</div>
285293
<span className='text-[12px] text-[var(--text-secondary)]'>
286294
{tokenCount.toLocaleString()}

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx‎

Lines changed: 2 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import {
3636
import { Input } from '@/components/ui/input'
3737
import { SearchHighlight } from '@/components/ui/search-highlight'
3838
import { Skeleton } from '@/components/ui/skeleton'
39+
import { formatAbsoluteDate, formatRelativeTime } from '@/lib/core/utils/formatting'
3940
import type { ChunkData } from '@/lib/knowledge/types'
4041
import {
4142
ChunkContextMenu,
@@ -58,55 +59,6 @@ import {
5859

5960
const logger = createLogger('Document')
6061

61-
/**
62-
* Formats a date string to relative time (e.g., "2h ago", "3d ago")
63-
*/
64-
function formatRelativeTime(dateString: string): string {
65-
const date = new Date(dateString)
66-
const now = new Date()
67-
const diffInSeconds = Math.floor((now.getTime() - date.getTime()) / 1000)
68-
69-
if (diffInSeconds < 60) {
70-
return 'just now'
71-
}
72-
if (diffInSeconds < 3600) {
73-
const minutes = Math.floor(diffInSeconds / 60)
74-
return `${minutes}m ago`
75-
}
76-
if (diffInSeconds < 86400) {
77-
const hours = Math.floor(diffInSeconds / 3600)
78-
return `${hours}h ago`
79-
}
80-
if (diffInSeconds < 604800) {
81-
const days = Math.floor(diffInSeconds / 86400)
82-
return `${days}d ago`
83-
}
84-
if (diffInSeconds < 2592000) {
85-
const weeks = Math.floor(diffInSeconds / 604800)
86-
return `${weeks}w ago`
87-
}
88-
if (diffInSeconds < 31536000) {
89-
const months = Math.floor(diffInSeconds / 2592000)
90-
return `${months}mo ago`
91-
}
92-
const years = Math.floor(diffInSeconds / 31536000)
93-
return `${years}y ago`
94-
}
95-
96-
/**
97-
* Formats a date string to absolute format for tooltip display
98-
*/
99-
function formatAbsoluteDate(dateString: string): string {
100-
const date = new Date(dateString)
101-
return date.toLocaleDateString('en-US', {
102-
year: 'numeric',
103-
month: 'short',
104-
day: 'numeric',
105-
hour: '2-digit',
106-
minute: '2-digit',
107-
})
108-
}
109-
11062
interface DocumentProps {
11163
knowledgeBaseId: string
11264
documentId: string
@@ -304,7 +256,6 @@ export function Document({
304256

305257
const [searchQuery, setSearchQuery] = useState('')
306258
const [debouncedSearchQuery, setDebouncedSearchQuery] = useState('')
307-
const [isSearching, setIsSearching] = useState(false)
308259

309260
const {
310261
chunks: initialChunks,
@@ -344,7 +295,6 @@ export function Document({
344295
const handler = setTimeout(() => {
345296
startTransition(() => {
346297
setDebouncedSearchQuery(searchQuery)
347-
setIsSearching(searchQuery.trim().length > 0)
348298
})
349299
}, 200)
350300

@@ -353,6 +303,7 @@ export function Document({
353303
}
354304
}, [searchQuery])
355305

306+
const isSearching = debouncedSearchQuery.trim().length > 0
356307
const showingSearch = isSearching && searchQuery.trim().length > 0 && searchResults.length > 0
357308
const SEARCH_PAGE_SIZE = 50
358309
const maxSearchPages = Math.ceil(searchResults.length / SEARCH_PAGE_SIZE)

0 commit comments

Comments
 (0)