Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
495 changes: 290 additions & 205 deletions packages/node-type-registry/src/blueprint-types.generated.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion packages/node-type-registry/src/codegen/generate-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1139,7 +1139,7 @@ function buildProgram(meta?: MetaTableInfo[]): string {
statements.push(buildTriggerConditionInterface());

// -- Parameter interfaces grouped by category --
const categoryOrder = ['check', 'data', 'search', 'authz', 'relation', 'view'];
const categoryOrder = ['billing', 'check', 'data', 'limit', 'search', 'job', 'process', 'authz', 'relation', 'view'];
for (const cat of categoryOrder) {
const nts = categories.get(cat);
if (!nts || nts.length === 0) continue;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import type { NodeTypeDefinition } from '../types';

export const DataAggregateLimitCounter: NodeTypeDefinition = {
name: 'DataAggregateLimitCounter',
export const LimitAggregate: NodeTypeDefinition = {
name: 'LimitAggregate',
slug: 'data_aggregate_limit_counter',
category: 'data',
category: 'limit',
display_name: 'Aggregate Limit Counter',
description:
'Declaratively attaches aggregate limit-tracking triggers to a table. On INSERT the named limit is incremented per entity; on DELETE it is decremented. Uses org_limit_aggregates_inc/dec for per-entity (org-level) aggregate limits rather than per-user limits. Requires a provisioned limits_module for the target database.',
Expand Down
6 changes: 3 additions & 3 deletions packages/node-type-registry/src/data/data-billing-meter.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import type { NodeTypeDefinition } from '../types';

export const DataBillingMeter: NodeTypeDefinition = {
name: 'DataBillingMeter',
export const BillingMeter: NodeTypeDefinition = {
name: 'BillingMeter',
slug: 'data_billing_meter',
category: 'data',
category: 'billing',
display_name: 'Billing Meter',
description:
'Declaratively attaches billing usage-recording triggers to a table. On INSERT the named meter is incremented via record_usage; on DELETE it is decremented (reversal). On UPDATE, if the entity_field changes, the old entity is decremented and the new entity is incremented. Requires a provisioned billing_module for the target database.',
Expand Down
10 changes: 5 additions & 5 deletions packages/node-type-registry/src/data/data-chunks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,20 @@ import type { NodeTypeDefinition } from '../types';
* - RLS policies inherited from parent
* - Optional job trigger for automatic chunking on INSERT/UPDATE
*
* This node is also composed internally by DataFileEmbedding (enabled by
* This node is also composed internally by ProcessFileEmbedding (enabled by
* default in extract mode). Use it standalone when you want a chunks table
* without the full file-embedding pipeline.
*/
export const DataChunks: NodeTypeDefinition = {
name: 'DataChunks',
export const ProcessChunks: NodeTypeDefinition = {
name: 'ProcessChunks',
slug: 'data_chunks',
category: 'data',
category: 'process',
display_name: 'Chunks',
description:
'Creates a chunked-embedding child table for any parent table. ' +
'Provisions the chunks table with content, chunk_index, embedding vector, ' +
'metadata, HNSW index, inherited RLS, and optional job trigger for ' +
'automatic text splitting. Composed internally by DataFileEmbedding ' +
'automatic text splitting. Composed internally by ProcessFileEmbedding ' +
'(enabled by default in extract mode) but can also be used standalone.',
parameter_schema: {
type: 'object',
Expand Down
6 changes: 3 additions & 3 deletions packages/node-type-registry/src/data/data-feature-flag.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import type { NodeTypeDefinition } from '../types';

export const DataFeatureFlag: NodeTypeDefinition = {
name: 'DataFeatureFlag',
export const LimitFeatureFlag: NodeTypeDefinition = {
name: 'LimitFeatureFlag',
slug: 'data_feature_flag',
category: 'data',
category: 'limit',
display_name: 'Feature Flag',
description:
'Gates a table behind a feature flag backed by the cap tables. Attaches a BEFORE INSERT trigger that checks whether the named feature cap value is > 0. Features are modeled as caps with max=0 (disabled) or max=1 (enabled) in limit_caps / limit_caps_defaults tables. Resolution: COALESCE(per-entity cap, scope default, 0).',
Expand Down
14 changes: 7 additions & 7 deletions packages/node-type-registry/src/data/data-file-embedding.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
import type { NodeTypeDefinition } from '../types';

export const DataFileEmbedding: NodeTypeDefinition = {
name: 'DataFileEmbedding',
export const ProcessFileEmbedding: NodeTypeDefinition = {
name: 'ProcessFileEmbedding',
slug: 'data_file_embedding',
category: 'data',
category: 'process',
display_name: 'File Embedding',
description:
'Generic, MIME-scoped embedding node for file tables. Supports two modes: ' +
'direct (whole-file to single vector, e.g. CLIP for images) when extraction ' +
'is omitted, or extract (file to text to chunks to per-chunk vectors) when ' +
'extraction config is provided. Composes SearchVector + DataJobTrigger + ' +
'DataChunks (enabled by default in extract mode) internally. Multiple ' +
'extraction config is provided. Composes SearchVector + JobTrigger + ' +
'ProcessChunks (enabled by default in extract mode) internally. Multiple ' +
'instances can coexist on the same table with different MIME scopes, field ' +
'names, and embedding strategies.',
parameter_schema: {
Expand Down Expand Up @@ -122,14 +122,14 @@ export const DataFileEmbedding: NodeTypeDefinition = {
include_chunks: {
type: 'boolean',
description:
'Whether to create a chunks table via DataChunks. Defaults to true ' +
'Whether to create a chunks table via ProcessChunks. Defaults to true ' +
'when extraction is provided, false in direct mode. Set explicitly ' +
'to override.',
},
chunks: {
type: 'object',
description:
'Chunking configuration passed through to DataChunks. When ' +
'Chunking configuration passed through to ProcessChunks. When ' +
'include_chunks is true (or defaults to true in extract mode), these ' +
'params configure the chunks table, embedding dimensions, strategy, etc.',
properties: {
Expand Down
24 changes: 12 additions & 12 deletions packages/node-type-registry/src/data/data-image-embedding.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
import type { NodeTypeDefinition } from '../types';

/**
* Image-specific preset of DataFileEmbedding.
* Image-specific preset of ProcessFileEmbedding.
*
* At the SQL layer, data_image_embedding delegates entirely to
* data_file_embedding, merging image-specific defaults before forwarding.
* The parameter schema here is intentionally identical to DataFileEmbedding;
* The parameter schema here is intentionally identical to ProcessFileEmbedding;
* only the defaults differ (dimensions: 512, task: process_image_embedding,
* mime_patterns: ['image/%']).
*
* Kept as a separate node type for backward compatibility — existing
* blueprints that reference DataImageEmbedding continue to work unchanged.
* blueprints that reference ProcessImageEmbedding continue to work unchanged.
*/
export const DataImageEmbedding: NodeTypeDefinition = {
name: 'DataImageEmbedding',
export const ProcessImageEmbedding: NodeTypeDefinition = {
name: 'ProcessImageEmbedding',
slug: 'data_image_embedding',
category: 'data',
category: 'process',
display_name: 'Image Embedding',
description:
'Image-specific preset of DataFileEmbedding. Delegates to DataFileEmbedding ' +
'Image-specific preset of ProcessFileEmbedding. Delegates to ProcessFileEmbedding ' +
'with image-oriented defaults: dimensions=512 (CLIP), mime_patterns=[\'image/%\'], ' +
'task_identifier=\'process_image_embedding\', direct mode (no extraction). ' +
'Accepts all DataFileEmbedding parameters — any overrides are forwarded through.',
'Accepts all ProcessFileEmbedding parameters — any overrides are forwarded through.',
parameter_schema: {
type: 'object',
properties: {

// ── Vector config (passed through to DataFileEmbedding) ──────────
// ── Vector config (passed through to ProcessFileEmbedding) ──────────
field_name: {
type: 'string',
format: 'column-ref',
Expand Down Expand Up @@ -103,7 +103,7 @@ export const DataImageEmbedding: NodeTypeDefinition = {
extraction: {
type: 'object',
description:
'Text extraction configuration. Forwarded to DataFileEmbedding. ' +
'Text extraction configuration. Forwarded to ProcessFileEmbedding. ' +
'When present, enables extract mode (e.g., OCR for images).',
properties: {
text_field: {
Expand All @@ -121,11 +121,11 @@ export const DataImageEmbedding: NodeTypeDefinition = {
}
},

// ── Chunking config (optional — forwarded to DataFileEmbedding) ─
// ── Chunking config (optional — forwarded to ProcessFileEmbedding) ─
chunks: {
type: 'object',
description:
'Chunking configuration. Forwarded to DataFileEmbedding. ' +
'Chunking configuration. Forwarded to ProcessFileEmbedding. ' +
'Only meaningful when extraction is also provided.',
properties: {
content_field_name: {
Expand Down
6 changes: 3 additions & 3 deletions packages/node-type-registry/src/data/data-job-trigger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ const triggerConditionSchema = {
}
};

export const DataJobTrigger: NodeTypeDefinition = {
name: 'DataJobTrigger',
export const JobTrigger: NodeTypeDefinition = {
name: 'JobTrigger',
slug: 'data_job_trigger',
category: 'data',
category: 'job',
display_name: 'Job Trigger',
description: 'Dynamically creates PostgreSQL triggers that enqueue jobs via app_jobs.add_job() when table rows are inserted, updated, or deleted. Supports configurable payload strategies (full row, row ID, selected fields, or custom mapping), conditional firing via WHEN clauses, watched field changes, and extended job options (queue, priority, delay, max attempts).',
parameter_schema: {
Expand Down
6 changes: 3 additions & 3 deletions packages/node-type-registry/src/data/data-limit-counter.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import type { NodeTypeDefinition } from '../types';

export const DataLimitCounter: NodeTypeDefinition = {
name: 'DataLimitCounter',
export const LimitCounter: NodeTypeDefinition = {
name: 'LimitCounter',
slug: 'data_limit_counter',
category: 'data',
category: 'limit',
display_name: 'Limit Counter',
description:
'Declaratively attaches limit-tracking triggers to a table. On INSERT the named limit is incremented; on DELETE it is decremented. Requires a provisioned limits_module for the target scope.',
Expand Down
18 changes: 10 additions & 8 deletions packages/node-type-registry/src/data/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,27 @@ export { CheckGreaterThan } from './check-greater-than';
export { CheckLessThan } from './check-less-than';
export { CheckNotEqual } from './check-not-equal';
export { CheckOneOf } from './check-one-of';
export { DataAggregateLimitCounter } from './data-aggregate-limit-counter';
export { DataBillingMeter } from './data-billing-meter';
export { LimitAggregate } from './data-aggregate-limit-counter';
export { BillingMeter } from './data-billing-meter';
export { DataBulk } from './data-bulk';
export { DataChunks } from './data-chunks';
export { ProcessChunks } from './data-chunks';
export { DataCompositeField } from './data-composite-field';
export { DataDirectOwner } from './data-direct-owner';
export { DataEntityMembership } from './data-entity-membership';
export { DataFileEmbedding } from './data-file-embedding';
export { DataFeatureFlag } from './data-feature-flag';
export { ProcessFileEmbedding } from './data-file-embedding';
export { LimitFeatureFlag } from './data-feature-flag';
export { DataForceCurrentUser } from './data-force-current-user';
export { DataId } from './data-id';
export { DataImageEmbedding } from './data-image-embedding';
export { ProcessImageEmbedding } from './data-image-embedding';
export { DataImmutableFields } from './data-immutable-fields';
export { DataInflection } from './data-inflection';
export { DataInheritFromParent } from './data-inherit-from-parent';
export { DataJobTrigger } from './data-job-trigger';
export { DataLimitCounter } from './data-limit-counter';
export { JobTrigger } from './data-job-trigger';
export { LimitCounter } from './data-limit-counter';
export { DataJsonb } from './data-jsonb';
export { DataOwnedFields } from './data-owned-fields';
export { ProcessExtraction } from './process-extraction';
export { ProcessImageVersions } from './process-image-versions';
export { DataOwnershipInEntity } from './data-ownership-in-entity';
export { DataPeoplestamps } from './data-peoplestamps';
export { DataPublishable } from './data-publishable';
Expand Down
114 changes: 114 additions & 0 deletions packages/node-type-registry/src/data/process-extraction.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import type { NodeTypeDefinition } from '../types';

/**
* File extraction processing node.
*
* Composes a JobTrigger that fires when a file transitions to status = 'uploaded'
* (or on INSERT if confirm_upload is not enabled). The trigger enqueues a
* text-extraction job that converts the file contents (PDF, DOCX, HTML, etc.)
* into plain text or markdown, storing the result in configurable output fields.
*
* The extraction worker is external (Knative function) — this node only creates
* the trigger infrastructure and output fields. The worker calls back into the
* database to write extracted text and metadata.
*/
export const ProcessExtraction: NodeTypeDefinition = {
name: 'ProcessExtraction',
slug: 'process_extraction',
category: 'process',
display_name: 'File Extraction',
description:
'Creates extraction output fields and a job trigger for file text extraction. ' +
'Fires when a file is uploaded (status = \'uploaded\') or on INSERT. ' +
'The external worker extracts text/metadata from the file (PDF, DOCX, HTML, etc.) ' +
'and writes the result back to the configured output fields. ' +
'Typically used upstream of ProcessFileEmbedding or ProcessChunks.',
parameter_schema: {
type: 'object',
properties: {

// ── Output fields ─────────────────────────────────────────────
text_field: {
type: 'string',
format: 'column-ref',
description: 'Field to store extracted text/markdown',
default: 'extracted_text'
},
metadata_field: {
type: 'string',
format: 'column-ref',
description: 'JSONB field for extraction metadata (page count, language, etc.)',
default: 'extracted_metadata'
},

// ── MIME scoping ──────────────────────────────────────────────
mime_patterns: {
type: 'array',
items: { type: 'string' },
description:
'MIME type LIKE patterns to match. Multiple patterns are OR\'d together. ' +
'Examples: [\'application/pdf\', \'text/%\'], [\'application/vnd.openxmlformats%\'].',
default: ['application/pdf', 'text/%']
},

// ── Job routing ───────────────────────────────────────────────
task_identifier: {
type: 'string',
description: 'Job task identifier for the extraction worker',
default: 'extract_file_text'
},
events: {
type: 'array',
items: { type: 'string', enum: ['INSERT', 'UPDATE'] },
description: 'Trigger events that fire the job',
default: ['INSERT']
},
payload_custom: {
type: 'object',
additionalProperties: { type: 'string', format: 'column-ref' },
description: 'Custom payload key-to-column mapping for the job trigger',
default: {
file_id: 'id',
key: 'key',
mime_type: 'mime_type',
bucket_id: 'bucket_id'
}
},
trigger_conditions: {
description:
'Additional compound conditions beyond MIME filtering. ' +
'Merged with the auto-generated MIME conditions via AND. ' +
'Use this to add status checks (e.g., status = \'uploaded\').',
'x-codegen-type': 'TriggerCondition | TriggerCondition[]',
oneOf: [
{ $ref: '#/$defs/triggerCondition' },
{ type: 'array', items: { $ref: '#/$defs/triggerCondition' } }
]
},

// ── Job options ───────────────────────────────────────────────
queue_name: {
type: 'string',
description: 'Job queue name for extraction tasks',
default: 'extraction'
},
max_attempts: {
type: 'integer',
description: 'Maximum number of retry attempts',
default: 5
},
priority: {
type: 'integer',
description: 'Job priority (lower = higher priority)',
default: 0
}
}
},
tags: [
'extraction',
'files',
'processing',
'jobs',
'text'
]
};
Loading
Loading