diff --git a/packages/database-jobs/Makefile b/packages/database-jobs/Makefile index 85442957..5bc52142 100644 --- a/packages/database-jobs/Makefile +++ b/packages/database-jobs/Makefile @@ -1,5 +1,5 @@ EXTENSION = pgpm-database-jobs -DATA = sql/pgpm-database-jobs--0.22.0.sql +DATA = sql/pgpm-database-jobs--0.26.0.sql PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/packages/database-jobs/__tests__/__snapshots__/jobs.test.ts.snap b/packages/database-jobs/__tests__/__snapshots__/jobs.test.ts.snap index 2a9114b7..46f25498 100644 --- a/packages/database-jobs/__tests__/__snapshots__/jobs.test.ts.snap +++ b/packages/database-jobs/__tests__/__snapshots__/jobs.test.ts.snap @@ -5,6 +5,7 @@ exports[`scheduled jobs schedule jobs 1`] = ` "actor_id": null, "attempts": 0, "database_id": "5b720132-17d5-424d-9bcb-ee7b17c13d43", + "entity_id": null, "id": "1", "is_available": true, "key": null, diff --git a/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_job.sql b/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_job.sql index 8165d50e..ccd99621 100644 --- a/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_job.sql +++ b/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_job.sql @@ -13,7 +13,8 @@ CREATE FUNCTION app_jobs.add_job ( queue_name text DEFAULT NULL, run_at timestamptz DEFAULT now(), max_attempts integer DEFAULT 25, - priority integer DEFAULT 0 + priority integer DEFAULT 0, + entity_id uuid DEFAULT NULL ) RETURNS app_jobs.jobs AS $$ @@ -31,6 +32,7 @@ BEGIN INSERT INTO app_jobs.jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -41,6 +43,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_job.entity_id, identifier, coalesce(payload, '{}'::json), queue_name, @@ -84,6 +87,7 @@ BEGIN INSERT INTO app_jobs.jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -93,6 +97,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_job.entity_id, identifier, payload, queue_name, diff --git a/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql b/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql index c5c730c9..e9d86777 100644 --- a/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql +++ b/packages/database-jobs/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql @@ -14,7 +14,8 @@ CREATE FUNCTION app_jobs.add_scheduled_job( job_key text DEFAULT NULL, queue_name text DEFAULT NULL, max_attempts integer DEFAULT 25, - priority integer DEFAULT 0 + priority integer DEFAULT 0, + entity_id uuid DEFAULT NULL ) RETURNS app_jobs.scheduled_jobs AS $$ @@ -32,6 +33,7 @@ BEGIN INSERT INTO app_jobs.scheduled_jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -42,6 +44,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_scheduled_job.entity_id, identifier, coalesce(payload, '{}'::json), queue_name, @@ -81,6 +84,7 @@ BEGIN INSERT INTO app_jobs.scheduled_jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -90,6 +94,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_scheduled_job.entity_id, identifier, payload, queue_name, diff --git a/packages/database-jobs/deploy/schemas/app_jobs/procedures/run_scheduled_job.sql b/packages/database-jobs/deploy/schemas/app_jobs/procedures/run_scheduled_job.sql index 7a4d5831..7bdb4399 100644 --- a/packages/database-jobs/deploy/schemas/app_jobs/procedures/run_scheduled_job.sql +++ b/packages/database-jobs/deploy/schemas/app_jobs/procedures/run_scheduled_job.sql @@ -42,6 +42,7 @@ BEGIN INSERT INTO app_jobs.jobs ( database_id, actor_id, + entity_id, queue_name, task_identifier, payload, @@ -51,6 +52,7 @@ BEGIN ) SELECT database_id, actor_id, + entity_id, queue_name, task_identifier, payload, diff --git a/packages/database-jobs/deploy/schemas/app_jobs/tables/jobs/table.sql b/packages/database-jobs/deploy/schemas/app_jobs/tables/jobs/table.sql index 3083bb68..92933621 100644 --- a/packages/database-jobs/deploy/schemas/app_jobs/tables/jobs/table.sql +++ b/packages/database-jobs/deploy/schemas/app_jobs/tables/jobs/table.sql @@ -6,6 +6,7 @@ CREATE TABLE app_jobs.jobs ( id bigserial PRIMARY KEY, database_id uuid, actor_id uuid, + entity_id uuid, queue_name text DEFAULT NULL, task_identifier text NOT NULL, payload json DEFAULT '{}' ::json NOT NULL, @@ -30,6 +31,7 @@ COMMENT ON TABLE app_jobs.jobs IS 'Background job queue: each row is a pending o COMMENT ON COLUMN app_jobs.jobs.id IS 'Auto-incrementing job identifier'; COMMENT ON COLUMN app_jobs.jobs.database_id IS 'Database this job belongs to (nullable for system-level jobs without tenant context)'; COMMENT ON COLUMN app_jobs.jobs.actor_id IS 'User who triggered this job, read from JWT claims at enqueue time'; +COMMENT ON COLUMN app_jobs.jobs.entity_id IS 'Entity (org/team) this job is scoped to for billing; NULL means platform-level (resolved via database_id → owner_id)'; COMMENT ON COLUMN app_jobs.jobs.queue_name IS 'Name of the queue this job belongs to; used for worker routing and concurrency control'; COMMENT ON COLUMN app_jobs.jobs.task_identifier IS 'Identifier for the task type (maps to a worker handler function)'; COMMENT ON COLUMN app_jobs.jobs.payload IS 'JSON payload of arguments passed to the task handler'; diff --git a/packages/database-jobs/deploy/schemas/app_jobs/tables/scheduled_jobs/table.sql b/packages/database-jobs/deploy/schemas/app_jobs/tables/scheduled_jobs/table.sql index 74ce5584..bbf7820a 100644 --- a/packages/database-jobs/deploy/schemas/app_jobs/tables/scheduled_jobs/table.sql +++ b/packages/database-jobs/deploy/schemas/app_jobs/tables/scheduled_jobs/table.sql @@ -6,6 +6,7 @@ CREATE TABLE app_jobs.scheduled_jobs ( id bigserial PRIMARY KEY, database_id uuid, actor_id uuid, + entity_id uuid, queue_name text DEFAULT NULL, task_identifier text NOT NULL, payload json DEFAULT '{}' ::json NOT NULL, @@ -29,6 +30,7 @@ COMMENT ON TABLE app_jobs.scheduled_jobs IS 'Recurring/cron-style job definition COMMENT ON COLUMN app_jobs.scheduled_jobs.id IS 'Auto-incrementing scheduled job identifier'; COMMENT ON COLUMN app_jobs.scheduled_jobs.database_id IS 'Database this scheduled job belongs to (nullable for system-level schedules without tenant context)'; COMMENT ON COLUMN app_jobs.scheduled_jobs.actor_id IS 'User who created this scheduled job, read from JWT claims at creation time'; +COMMENT ON COLUMN app_jobs.scheduled_jobs.entity_id IS 'Entity (org/team) this scheduled job is scoped to for billing; NULL means platform-level (resolved via database_id → owner_id)'; COMMENT ON COLUMN app_jobs.scheduled_jobs.queue_name IS 'Name of the queue spawned jobs are placed into'; COMMENT ON COLUMN app_jobs.scheduled_jobs.task_identifier IS 'Task type identifier for spawned jobs'; COMMENT ON COLUMN app_jobs.scheduled_jobs.payload IS 'JSON payload passed to each spawned job'; diff --git a/packages/database-jobs/package.json b/packages/database-jobs/package.json index 1ef57643..10b0a793 100644 --- a/packages/database-jobs/package.json +++ b/packages/database-jobs/package.json @@ -35,4 +35,4 @@ "bugs": { "url": "https://github.com/constructive-io/pgpm-modules/issues" } -} +} \ No newline at end of file diff --git a/packages/database-jobs/pgpm-database-jobs.control b/packages/database-jobs/pgpm-database-jobs.control index 3ccad9d7..d1944d01 100644 --- a/packages/database-jobs/pgpm-database-jobs.control +++ b/packages/database-jobs/pgpm-database-jobs.control @@ -1,6 +1,6 @@ # pgpm-database-jobs extension comment = 'pgpm-database-jobs extension' -default_version = '0.22.0' +default_version = '0.26.0' module_pathname = '$libdir/pgpm-database-jobs' requires = 'plpgsql,pgcrypto,pgpm-verify,pgpm-jwt-claims' relocatable = false diff --git a/packages/database-jobs/sql/pgpm-database-jobs--0.22.0.sql b/packages/database-jobs/sql/pgpm-database-jobs--0.26.0.sql similarity index 87% rename from packages/database-jobs/sql/pgpm-database-jobs--0.22.0.sql rename to packages/database-jobs/sql/pgpm-database-jobs--0.26.0.sql index ccf7a4db..390c4b7e 100644 --- a/packages/database-jobs/sql/pgpm-database-jobs--0.22.0.sql +++ b/packages/database-jobs/sql/pgpm-database-jobs--0.26.0.sql @@ -53,7 +53,9 @@ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; COMMENT ON FUNCTION app_jobs.tg_add_job_with_row IS 'Useful shortcut to create a job on insert or update. Pass the task name as the trigger argument, and the record data will automatically be available on the JSON payload.'; -CREATE FUNCTION app_jobs.json_build_object_apply(arguments text[]) RETURNS pg_catalog.json AS $EOFCODE$ +CREATE FUNCTION app_jobs.json_build_object_apply( + arguments text[] +) RETURNS pg_catalog.json AS $EOFCODE$ DECLARE arg text; _sql text; @@ -116,6 +118,7 @@ CREATE TABLE app_jobs.scheduled_jobs ( id bigserial PRIMARY KEY, database_id uuid, actor_id uuid, + entity_id uuid, queue_name text DEFAULT NULL, task_identifier text NOT NULL, payload pg_catalog.json DEFAULT '{}'::json NOT NULL, @@ -143,6 +146,8 @@ COMMENT ON COLUMN app_jobs.scheduled_jobs.database_id IS 'Database this schedule COMMENT ON COLUMN app_jobs.scheduled_jobs.actor_id IS 'User who created this scheduled job, read from JWT claims at creation time'; +COMMENT ON COLUMN app_jobs.scheduled_jobs.entity_id IS 'Entity (org/team) this scheduled job is scoped to for billing; NULL means platform-level (resolved via database_id → owner_id)'; + COMMENT ON COLUMN app_jobs.scheduled_jobs.queue_name IS 'Name of the queue spawned jobs are placed into'; COMMENT ON COLUMN app_jobs.scheduled_jobs.task_identifier IS 'Task type identifier for spawned jobs'; @@ -189,6 +194,7 @@ CREATE TABLE app_jobs.jobs ( id bigserial PRIMARY KEY, database_id uuid, actor_id uuid, + entity_id uuid, queue_name text DEFAULT NULL, task_identifier text NOT NULL, payload pg_catalog.json DEFAULT '{}'::json NOT NULL, @@ -218,6 +224,8 @@ COMMENT ON COLUMN app_jobs.jobs.database_id IS 'Database this job belongs to (nu COMMENT ON COLUMN app_jobs.jobs.actor_id IS 'User who triggered this job, read from JWT claims at enqueue time'; +COMMENT ON COLUMN app_jobs.jobs.entity_id IS 'Entity (org/team) this job is scoped to for billing; NULL means platform-level (resolved via database_id → owner_id)'; + COMMENT ON COLUMN app_jobs.jobs.queue_name IS 'Name of the queue this job belongs to; used for worker routing and concurrency control'; COMMENT ON COLUMN app_jobs.jobs.task_identifier IS 'Identifier for the task type (maps to a worker handler function)'; @@ -366,7 +374,10 @@ CREATE INDEX job_queues_locked_by_idx ON app_jobs.job_queues (locked_by); GRANT SELECT, INSERT, UPDATE, DELETE ON app_jobs.job_queues TO administrator; -CREATE FUNCTION app_jobs.run_scheduled_job(id bigint, job_expiry interval DEFAULT '1 hours') RETURNS app_jobs.jobs AS $EOFCODE$ +CREATE FUNCTION app_jobs.run_scheduled_job( + id bigint, + job_expiry interval DEFAULT '1 hours' +) RETURNS app_jobs.jobs AS $EOFCODE$ DECLARE j app_jobs.jobs; last_id bigint; @@ -402,6 +413,7 @@ BEGIN INSERT INTO app_jobs.jobs ( database_id, actor_id, + entity_id, queue_name, task_identifier, payload, @@ -411,6 +423,7 @@ BEGIN ) SELECT database_id, actor_id, + entity_id, queue_name, task_identifier, payload, @@ -435,7 +448,13 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE; -CREATE FUNCTION app_jobs.reschedule_jobs(job_ids bigint[], run_at timestamptz DEFAULT NULL, priority int DEFAULT NULL, attempts int DEFAULT NULL, max_attempts int DEFAULT NULL) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$ +CREATE FUNCTION app_jobs.reschedule_jobs( + job_ids bigint[], + run_at timestamptz DEFAULT NULL, + priority int DEFAULT NULL, + attempts int DEFAULT NULL, + max_attempts int DEFAULT NULL +) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$ UPDATE app_jobs.jobs SET @@ -451,7 +470,10 @@ CREATE FUNCTION app_jobs.reschedule_jobs(job_ids bigint[], run_at timestamptz DE *; $EOFCODE$; -CREATE FUNCTION app_jobs.release_scheduled_jobs(worker_id text, ids bigint[] DEFAULT NULL) RETURNS void AS $EOFCODE$ +CREATE FUNCTION app_jobs.release_scheduled_jobs( + worker_id text, + ids bigint[] DEFAULT NULL +) RETURNS void AS $EOFCODE$ DECLARE BEGIN -- clear the scheduled job @@ -467,7 +489,9 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE; -CREATE FUNCTION app_jobs.release_jobs(worker_id text) RETURNS void AS $EOFCODE$ +CREATE FUNCTION app_jobs.release_jobs( + worker_id text +) RETURNS void AS $EOFCODE$ DECLARE BEGIN -- clear the job @@ -490,7 +514,10 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE; -CREATE FUNCTION app_jobs.permanently_fail_jobs(job_ids bigint[], error_message text DEFAULT NULL) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$ +CREATE FUNCTION app_jobs.permanently_fail_jobs( + job_ids bigint[], + error_message text DEFAULT NULL +) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$ UPDATE app_jobs.jobs SET @@ -504,7 +531,10 @@ CREATE FUNCTION app_jobs.permanently_fail_jobs(job_ids bigint[], error_message t *; $EOFCODE$; -CREATE FUNCTION app_jobs.get_scheduled_job(worker_id text, task_identifiers text[] DEFAULT NULL) RETURNS app_jobs.scheduled_jobs LANGUAGE plpgsql AS $EOFCODE$ +CREATE FUNCTION app_jobs.get_scheduled_job( + worker_id text, + task_identifiers text[] DEFAULT NULL +) RETURNS app_jobs.scheduled_jobs LANGUAGE plpgsql AS $EOFCODE$ DECLARE v_job_id bigint; v_row app_jobs.scheduled_jobs; @@ -556,7 +586,11 @@ BEGIN END; $EOFCODE$; -CREATE FUNCTION app_jobs.get_job(worker_id text, task_identifiers text[] DEFAULT NULL, job_expiry interval DEFAULT '4 hours') RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$ +CREATE FUNCTION app_jobs.get_job( + worker_id text, + task_identifiers text[] DEFAULT NULL, + job_expiry interval DEFAULT '4 hours' +) RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$ DECLARE v_job_id bigint; v_queue_name text; @@ -611,7 +645,11 @@ BEGIN END; $EOFCODE$; -CREATE FUNCTION app_jobs.fail_job(worker_id text, job_id bigint, error_message text) RETURNS app_jobs.jobs LANGUAGE plpgsql STRICT AS $EOFCODE$ +CREATE FUNCTION app_jobs.fail_job( + worker_id text, + job_id bigint, + error_message text +) RETURNS app_jobs.jobs LANGUAGE plpgsql STRICT AS $EOFCODE$ DECLARE v_row app_jobs.jobs; BEGIN @@ -641,7 +679,9 @@ BEGIN END; $EOFCODE$; -CREATE FUNCTION app_jobs.complete_jobs(job_ids bigint[]) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$ +CREATE FUNCTION app_jobs.complete_jobs( + job_ids bigint[] +) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$ DELETE FROM app_jobs.jobs WHERE id = ANY (job_ids) AND (locked_by IS NULL @@ -650,7 +690,10 @@ CREATE FUNCTION app_jobs.complete_jobs(job_ids bigint[]) RETURNS SETOF app_jobs. *; $EOFCODE$; -CREATE FUNCTION app_jobs.complete_job(worker_id text, job_id bigint) RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$ +CREATE FUNCTION app_jobs.complete_job( + worker_id text, + job_id bigint +) RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$ DECLARE v_row app_jobs.jobs; BEGIN @@ -672,7 +715,16 @@ BEGIN END; $EOFCODE$; -CREATE FUNCTION app_jobs.add_scheduled_job(identifier text, payload pg_catalog.json DEFAULT '{}'::json, schedule_info pg_catalog.json DEFAULT '{}'::json, job_key text DEFAULT NULL, queue_name text DEFAULT NULL, max_attempts int DEFAULT 25, priority int DEFAULT 0) RETURNS app_jobs.scheduled_jobs AS $EOFCODE$ +CREATE FUNCTION app_jobs.add_scheduled_job( + identifier text, + payload pg_catalog.json DEFAULT '{}'::json, + schedule_info pg_catalog.json DEFAULT '{}'::json, + job_key text DEFAULT NULL, + queue_name text DEFAULT NULL, + max_attempts int DEFAULT 25, + priority int DEFAULT 0, + entity_id uuid DEFAULT NULL +) RETURNS app_jobs.scheduled_jobs AS $EOFCODE$ DECLARE v_job app_jobs.scheduled_jobs; v_database_id uuid; @@ -687,6 +739,7 @@ BEGIN INSERT INTO app_jobs.scheduled_jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -697,6 +750,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_scheduled_job.entity_id, identifier, coalesce(payload, '{}'::json), queue_name, @@ -736,6 +790,7 @@ BEGIN INSERT INTO app_jobs.scheduled_jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -745,6 +800,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_scheduled_job.entity_id, identifier, payload, queue_name, @@ -756,7 +812,16 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -CREATE FUNCTION app_jobs.add_job(identifier text, payload pg_catalog.json DEFAULT '{}'::json, job_key text DEFAULT NULL, queue_name text DEFAULT NULL, run_at timestamptz DEFAULT now(), max_attempts int DEFAULT 25, priority int DEFAULT 0) RETURNS app_jobs.jobs AS $EOFCODE$ +CREATE FUNCTION app_jobs.add_job( + identifier text, + payload pg_catalog.json DEFAULT '{}'::json, + job_key text DEFAULT NULL, + queue_name text DEFAULT NULL, + run_at timestamptz DEFAULT now(), + max_attempts int DEFAULT 25, + priority int DEFAULT 0, + entity_id uuid DEFAULT NULL +) RETURNS app_jobs.jobs AS $EOFCODE$ DECLARE v_job app_jobs.jobs; v_database_id uuid; @@ -771,6 +836,7 @@ BEGIN INSERT INTO app_jobs.jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -781,6 +847,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_job.entity_id, identifier, coalesce(payload, '{}'::json), queue_name, @@ -824,6 +891,7 @@ BEGIN INSERT INTO app_jobs.jobs ( database_id, actor_id, + entity_id, task_identifier, payload, queue_name, @@ -833,6 +901,7 @@ BEGIN ) VALUES ( v_database_id, v_actor_id, + add_job.entity_id, identifier, payload, queue_name, @@ -846,7 +915,9 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -CREATE FUNCTION app_jobs.remove_job(job_key text) RETURNS app_jobs.jobs LANGUAGE plpgsql STRICT AS $EOFCODE$ +CREATE FUNCTION app_jobs.remove_job( + job_key text +) RETURNS app_jobs.jobs LANGUAGE plpgsql STRICT AS $EOFCODE$ DECLARE v_job app_jobs.jobs; BEGIN @@ -871,7 +942,9 @@ BEGIN END; $EOFCODE$; -CREATE FUNCTION app_jobs.force_unlock_workers(worker_ids text[]) RETURNS void LANGUAGE sql VOLATILE AS $EOFCODE$ +CREATE FUNCTION app_jobs.force_unlock_workers( + worker_ids text[] +) RETURNS void LANGUAGE sql VOLATILE AS $EOFCODE$ UPDATE app_jobs.jobs SET locked_at = NULL, locked_by = NULL WHERE locked_by = ANY (worker_ids); diff --git a/packages/metaschema-modules/Makefile b/packages/metaschema-modules/Makefile index 83482a99..9f0f6cf5 100644 --- a/packages/metaschema-modules/Makefile +++ b/packages/metaschema-modules/Makefile @@ -1,5 +1,5 @@ EXTENSION = metaschema-modules -DATA = sql/metaschema-modules--0.15.5.sql +DATA = sql/metaschema-modules--0.26.0.sql PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/packages/metaschema-modules/__tests__/__snapshots__/modules.test.ts.snap b/packages/metaschema-modules/__tests__/__snapshots__/modules.test.ts.snap index b6dfa157..2cedd053 100644 --- a/packages/metaschema-modules/__tests__/__snapshots__/modules.test.ts.snap +++ b/packages/metaschema-modules/__tests__/__snapshots__/modules.test.ts.snap @@ -115,7 +115,7 @@ exports[`db_meta_modules should verify module table structures have database_id exports[`db_meta_modules should verify module tables have proper foreign key relationships 1`] = ` { - "constraintCount": 394602, + "constraintCount": 394604, "foreignTables": [ "database", "field", diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/agent_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/agent_module/table.sql index bc1a2ffe..2f53b0f2 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/agent_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/agent_module/table.sql @@ -41,6 +41,20 @@ CREATE TABLE metaschema_modules_public.agent_module ( -- Configurable security policies (NULL = use defaults based on membership_type) policies jsonb NULL, + -- Knowledge RAG config (dimensions, chunk_size, chunk_strategy, search_indexes, etc.) + -- NULL = use sensible defaults (768d, 1000 chars, paragraph, bm25) + knowledge_config jsonb NULL, + + -- Custom RLS policies for knowledge table (provisions.knowledge.policies) + -- NULL = use defaults (AuthzEntityMembership or AuthzAppMembership) + knowledge_policies jsonb NULL, + + -- Per-table provisions overrides from blueprint config. + -- Keys are table keys (thread, message, task, prompt, knowledge). + -- When a key is present, the module trigger skips default security for that table; + -- secure_table_provision applies the custom grants/policies instead. + provisions jsonb NULL, + -- Constraints CONSTRAINT agent_module_db_fkey FOREIGN KEY (database_id) REFERENCES metaschema_public.database (id) ON DELETE CASCADE, CONSTRAINT agent_module_schema_fkey FOREIGN KEY (schema_id) REFERENCES metaschema_public.schema (id) ON DELETE CASCADE, diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/billing_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/billing_module/table.sql index f13957e9..00dcdef9 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/billing_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/billing_module/table.sql @@ -31,6 +31,10 @@ CREATE TABLE metaschema_modules_public.billing_module ( meter_credits_table_id uuid NOT NULL DEFAULT uuid_nil(), meter_credits_table_name text NOT NULL DEFAULT '', + -- Meter sources table: maps billing meters to typed daily summary table columns + meter_sources_table_id uuid NOT NULL DEFAULT uuid_nil(), + meter_sources_table_name text NOT NULL DEFAULT '', + -- Generated functions record_usage_function text NOT NULL DEFAULT '', @@ -44,6 +48,7 @@ CREATE TABLE metaschema_modules_public.billing_module ( CONSTRAINT ledger_table_fkey FOREIGN KEY (ledger_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, CONSTRAINT balances_table_fkey FOREIGN KEY (balances_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, CONSTRAINT meter_credits_table_fkey FOREIGN KEY (meter_credits_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, + CONSTRAINT meter_sources_table_fkey FOREIGN KEY (meter_sources_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, CONSTRAINT billing_module_database_id_unique UNIQUE (database_id) ); diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/entity_type_provision/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/entity_type_provision/table.sql index ec8141cf..ebbe5dd3 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/entity_type_provision/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/entity_type_provision/table.sql @@ -47,8 +47,6 @@ CREATE TABLE metaschema_modules_public.entity_type_provision ( has_levels boolean NOT NULL DEFAULT false, - has_storage boolean NOT NULL DEFAULT false, - has_invites boolean NOT NULL DEFAULT false, has_invite_achievements boolean NOT NULL DEFAULT false, @@ -56,8 +54,9 @@ CREATE TABLE metaschema_modules_public.entity_type_provision ( -- ========================================================================= -- Storage configuration: JSON array of storage module definitions. -- Each element provisions a separate storage module with its own tables, - -- RLS policies, and feature flags. Only used when has_storage = true. - -- NULL = provision a single default storage module with default settings. + -- RLS policies, and feature flags. Presence triggers provisioning + -- (same inference model as namespaces, functions, agents). + -- NULL = do not provision. '[{}]' = provision one default storage module. -- ========================================================================= storage jsonb DEFAULT NULL, @@ -126,6 +125,8 @@ CREATE TABLE metaschema_modules_public.entity_type_provision ( out_namespaces_table_id uuid DEFAULT NULL, + out_namespace_events_table_id uuid DEFAULT NULL, + out_function_module_id uuid DEFAULT NULL, out_definitions_table_id uuid DEFAULT NULL, @@ -238,20 +239,14 @@ COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_levels IS Levels provide gamification/achievement tracking for members. When true, creates level steps, achievements, and level tables with security.'; -COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_storage IS - 'Whether to provision storage_module for this type. Defaults to false. - When true, creates {prefix}_buckets and {prefix}_files tables - with entity-scoped RLS (AuthzEntityMembership) using the entity''s membership_type. - Storage tables get owner_id FK to the entity table, so files are owned by the entity.'; - COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_invites IS 'Whether to provision invites_module for this type. Defaults to false. When true, the trigger inserts a row into invites_module which in turn (via insert_invites_module BEFORE INSERT) creates {prefix}_invites and {prefix}_claimed_invites tables plus the submit_{prefix}_invite_code() function. - Symmetric counterpart of has_storage. Re-provisioning is idempotent: the - UNIQUE (database_id, membership_type) constraint on invites_module combined with - ON CONFLICT DO NOTHING in the fan-out makes repeated INSERTs safe.'; + Re-provisioning is idempotent: the UNIQUE (database_id, membership_type) constraint + on invites_module combined with ON CONFLICT DO NOTHING in the fan-out makes + repeated INSERTs safe.'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_invite_achievements IS 'Whether to auto-attach an EventTracker to the claimed_invites table for invite-based @@ -330,10 +325,11 @@ COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_installed_ Populated by the trigger. Useful for verifying which modules were provisioned.'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.storage IS - 'Optional JSON array of storage module definitions. Each element provisions a separate - storage module with its own tables ({prefix}_{storage_key}_buckets/files), RLS policies, - and feature flags. Only used when has_storage = true; ignored otherwise. - NULL = provision a single default storage module with all defaults. + 'Optional JSON array of storage module definitions. Presence triggers provisioning + (same inference model as namespaces, functions, agents). + Each element provisions a separate storage module with its own tables + ({prefix}_{storage_key}_buckets/files), RLS policies, and feature flags. + NULL = do not provision storage. ''[{}]'' = provision one default storage module. Each array element recognizes (all optional): - storage_key (text) module discriminator, max 16 chars, lowercase snake_case. Defaults to ''default'' (omitted from table names). @@ -360,13 +356,13 @@ COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.storage IS storage := ''[{"has_path_shares": true, "buckets": [{"name": "documents"}]}, {"storage_key": "fn", "has_custom_keys": true, "buckets": [{"name": "functions"}]}]''::jsonb'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_storage_module_id IS - 'Output: the UUID of the storage_module row created for this entity type. Populated by the trigger when has_storage=true.'; + 'Output: the UUID of the storage_module row created for this entity type. Populated by the trigger when storage is non-NULL and non-empty.'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_buckets_table_id IS - 'Output: the UUID of the generated buckets table (e.g. data_room_buckets). Populated by the trigger when has_storage=true.'; + 'Output: the UUID of the generated buckets table (e.g. data_room_buckets). Populated by the trigger when storage is non-NULL and non-empty.'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_files_table_id IS - 'Output: the UUID of the generated files table (e.g. data_room_files). Populated by the trigger when has_storage=true.'; + 'Output: the UUID of the generated files table (e.g. data_room_files). Populated by the trigger when storage is non-NULL and non-empty.'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_invites_module_id IS 'Output: the UUID of the invites_module row created for this entity type. Populated by the trigger when has_invites=true. @@ -415,4 +411,8 @@ COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_namespaces 'Output: the UUID of the generated namespaces table (e.g. data_room_namespaces). Populated by the trigger when namespaces is non-NULL. NULL otherwise.'; +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_namespace_events_table_id IS + 'Output: the UUID of the generated namespace_events partitioned table (e.g. data_room_namespace_events). + Monthly partitioned, 12-month retention. Populated by the trigger when namespaces is non-NULL. NULL otherwise.'; + COMMIT; diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/function_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/function_module/table.sql index ad638a06..c4d72345 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/function_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/function_module/table.sql @@ -42,6 +42,12 @@ CREATE TABLE metaschema_modules_public.function_module ( -- {"$type": "AuthzEntityMembership", "privileges": ["select", "update"], "data": {...}} policies jsonb NULL, + -- Per-table provisions overrides from blueprint config. + -- Keys are table keys (definitions, invocations, execution_logs). + -- When a key is present, the module trigger skips default security for that table; + -- secure_table_provision applies the custom grants/policies instead. + provisions jsonb NULL, + -- Constraints CONSTRAINT function_module_db_fkey FOREIGN KEY (database_id) REFERENCES metaschema_public.database (id) ON DELETE CASCADE, CONSTRAINT function_module_schema_fkey FOREIGN KEY (schema_id) REFERENCES metaschema_public.schema (id) ON DELETE CASCADE, diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/graph_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/graph_module/table.sql index 72f0683e..90f6101c 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/graph_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/graph_module/table.sql @@ -27,7 +27,7 @@ CREATE TABLE metaschema_modules_public.graph_module ( -- Generated table IDs (populated by BEFORE INSERT trigger) graphs_table_id uuid NOT NULL DEFAULT uuid_nil(), executions_table_id uuid NOT NULL DEFAULT uuid_nil(), - exec_object_table_id uuid NOT NULL DEFAULT uuid_nil(), + outputs_table_id uuid NOT NULL DEFAULT uuid_nil(), -- API routing (get-or-create: if set, schema is added to this API; if NULL, no API is added) api_name text, @@ -45,6 +45,12 @@ CREATE TABLE metaschema_modules_public.graph_module ( -- {"$type": "AuthzEntityMembership", "privileges": ["select", "update"], "data": {...}} policies jsonb NULL, + -- Per-table provisions overrides from blueprint config. + -- Keys are table keys (graphs, executions, outputs). + -- When a key is present, the module trigger skips default security for that table; + -- secure_table_provision applies the custom grants/policies instead. + provisions jsonb NULL, + -- Timestamps created_at timestamptz NOT NULL DEFAULT now(), @@ -55,7 +61,7 @@ CREATE TABLE metaschema_modules_public.graph_module ( CONSTRAINT merkle_store_fkey FOREIGN KEY (merkle_store_module_id) REFERENCES metaschema_modules_public.merkle_store_module (id) ON DELETE CASCADE, CONSTRAINT graphs_table_fkey FOREIGN KEY (graphs_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, CONSTRAINT executions_table_fkey FOREIGN KEY (executions_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, - CONSTRAINT exec_object_table_fkey FOREIGN KEY (exec_object_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, + CONSTRAINT outputs_table_fkey FOREIGN KEY (outputs_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, CONSTRAINT graph_module_entity_table_fkey FOREIGN KEY (entity_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, -- Only one graph module per database + merkle store combination diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/inference_log_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/inference_log_module/table.sql index c12f26e9..fa4a830f 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/inference_log_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/inference_log_module/table.sql @@ -21,9 +21,14 @@ CREATE TABLE metaschema_modules_public.inference_log_module ( -- Partition lifecycle configuration "interval" text NOT NULL DEFAULT '1 month', - retention text NULL, + retention text NOT NULL DEFAULT '12 months', premake int NOT NULL DEFAULT 2, + -- Scope configuration: 'app' = per-app usage (actor_id RLS), 'platform' = tenant metering (database_id RLS) + scope text NOT NULL DEFAULT 'app', + actor_fk_table_id uuid NULL, + entity_fk_table_id uuid NULL, + prefix text NULL, CONSTRAINT db_fkey FOREIGN KEY (database_id) REFERENCES metaschema_public.database (id) ON DELETE CASCADE, @@ -31,7 +36,7 @@ CREATE TABLE metaschema_modules_public.inference_log_module ( CONSTRAINT private_schema_fkey FOREIGN KEY (private_schema_id) REFERENCES metaschema_public.schema (id) ON DELETE CASCADE, CONSTRAINT inference_log_table_fkey FOREIGN KEY (inference_log_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, CONSTRAINT usage_daily_table_fkey FOREIGN KEY (usage_daily_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, - CONSTRAINT inference_log_module_database_id_unique UNIQUE (database_id) + CONSTRAINT inference_log_module_database_id_prefix_unique UNIQUE NULLS NOT DISTINCT (database_id, prefix) ); CREATE INDEX inference_log_module_database_id_idx ON metaschema_modules_public.inference_log_module ( database_id ); diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/namespace_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/namespace_module/table.sql index 3aeb7344..6e0c2f8b 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/namespace_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/namespace_module/table.sql @@ -16,11 +16,13 @@ CREATE TABLE metaschema_modules_public.namespace_module ( public_schema_name text, private_schema_name text, - -- Generated table ID (populated by the generator) + -- Generated table IDs (populated by the generator) namespaces_table_id uuid NOT NULL DEFAULT uuid_nil(), + namespace_events_table_id uuid NOT NULL DEFAULT uuid_nil(), - -- Table name (input to the generator) + -- Table names (input to the generator) namespaces_table_name text NOT NULL DEFAULT 'namespaces', + namespace_events_table_name text NOT NULL DEFAULT 'namespace_events', -- API routing (get-or-create: if set, schema is added to this API; if NULL, no API is added) api_name text, @@ -38,11 +40,18 @@ CREATE TABLE metaschema_modules_public.namespace_module ( -- {"$type": "AuthzEntityMembership", "privileges": ["select", "update"], "data": {...}} policies jsonb NULL, + -- Per-table provisions overrides from blueprint config. + -- Keys are table keys (namespaces, namespace_events). + -- When a key is present, the module trigger skips default security for that table; + -- secure_table_provision applies the custom grants/policies instead. + provisions jsonb NULL, + -- Constraints CONSTRAINT namespace_module_db_fkey FOREIGN KEY (database_id) REFERENCES metaschema_public.database (id) ON DELETE CASCADE, CONSTRAINT namespace_module_schema_fkey FOREIGN KEY (schema_id) REFERENCES metaschema_public.schema (id) ON DELETE CASCADE, CONSTRAINT namespace_module_private_schema_fkey FOREIGN KEY (private_schema_id) REFERENCES metaschema_public.schema (id) ON DELETE CASCADE, CONSTRAINT namespace_module_namespaces_table_fkey FOREIGN KEY (namespaces_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, + CONSTRAINT namespace_module_events_table_fkey FOREIGN KEY (namespace_events_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, CONSTRAINT namespace_module_entity_table_fkey FOREIGN KEY (entity_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE ); diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/storage_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/storage_module/table.sql index 7e80b5f1..4a1611a8 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/storage_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/storage_module/table.sql @@ -35,12 +35,11 @@ CREATE TABLE metaschema_modules_public.storage_module ( -- {"$type": "AuthzEntityMembership", "privileges": ["select", "update"], "data": {...}} policies jsonb NULL, - -- Per-table skip list for apply_storage_security default policies. - -- When a table role name ("files", "buckets") is listed here, - -- apply_storage_security skips its default policies for that table. - -- Used by entity_type_provision to mark tables whose policies are - -- supplied via provisions (secure_table_provision). - skip_default_policy_tables text[] NOT NULL DEFAULT '{}', + -- Per-table provisions overrides from blueprint config. + -- Keys are table keys (files, buckets). + -- When a key is present, the module trigger skips default security for that table; + -- secure_table_provision applies the custom grants/policies instead. + provisions jsonb NULL, -- Entity table for RLS (NULL for app-level storage, entity table for entity-scoped storage) entity_table_id uuid NULL, diff --git a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/user_auth_module/table.sql b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/user_auth_module/table.sql index 7952b72e..d4608cb0 100644 --- a/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/user_auth_module/table.sql +++ b/packages/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/user_auth_module/table.sql @@ -19,7 +19,7 @@ CREATE TABLE metaschema_modules_public.user_auth_module ( session_credentials_table_id uuid NOT NULL DEFAULT uuid_nil(), audits_table_id uuid NOT NULL DEFAULT uuid_nil(), - audits_table_name text NOT NULL DEFAULT 'audit_logs', + audits_table_name text NOT NULL DEFAULT 'audit_log_auth', -- api_id uuid NOT NULL REFERENCES services_public.apis (id), diff --git a/packages/metaschema-modules/metaschema-modules.control b/packages/metaschema-modules/metaschema-modules.control index e0ea0b88..5afb7d6d 100644 --- a/packages/metaschema-modules/metaschema-modules.control +++ b/packages/metaschema-modules/metaschema-modules.control @@ -1,6 +1,6 @@ # metaschema-modules extension comment = 'metaschema-modules extension' -default_version = '0.15.5' +default_version = '0.26.0' module_pathname = '$libdir/metaschema-modules' requires = 'plpgsql,uuid-ossp,metaschema-schema,services,pgpm-verify' relocatable = false diff --git a/packages/metaschema-modules/package.json b/packages/metaschema-modules/package.json index 4f2ae229..a7456bba 100644 --- a/packages/metaschema-modules/package.json +++ b/packages/metaschema-modules/package.json @@ -35,4 +35,4 @@ "bugs": { "url": "https://github.com/constructive-io/pgpm-modules/issues" } -} +} \ No newline at end of file diff --git a/packages/metaschema-modules/sql/metaschema-modules--0.15.5.sql b/packages/metaschema-modules/sql/metaschema-modules--0.26.0.sql similarity index 83% rename from packages/metaschema-modules/sql/metaschema-modules--0.15.5.sql rename to packages/metaschema-modules/sql/metaschema-modules--0.26.0.sql index 0c7cf0dd..1ade7ee4 100644 --- a/packages/metaschema-modules/sql/metaschema-modules--0.15.5.sql +++ b/packages/metaschema-modules/sql/metaschema-modules--0.26.0.sql @@ -894,7 +894,7 @@ CREATE TABLE metaschema_modules_public.user_auth_module ( sessions_table_id uuid NOT NULL DEFAULT uuid_nil(), session_credentials_table_id uuid NOT NULL DEFAULT uuid_nil(), audits_table_id uuid NOT NULL DEFAULT uuid_nil(), - audits_table_name text NOT NULL DEFAULT 'audit_logs', + audits_table_name text NOT NULL DEFAULT 'audit_log_auth', sign_in_function text NOT NULL DEFAULT 'sign_in', sign_up_function text NOT NULL DEFAULT 'sign_up', sign_out_function text NOT NULL DEFAULT 'sign_out', @@ -1459,7 +1459,7 @@ CREATE TABLE metaschema_modules_public.storage_module ( membership_type int DEFAULT NULL, storage_key text NOT NULL DEFAULT 'default', policies jsonb NULL, - skip_default_policy_tables text[] NOT NULL DEFAULT '{}', + provisions jsonb NULL, entity_table_id uuid NULL, endpoint text NULL, public_url_prefix text NULL, @@ -1532,10 +1532,13 @@ CREATE TABLE metaschema_modules_public.entity_type_provision ( has_limits boolean NOT NULL DEFAULT false, has_profiles boolean NOT NULL DEFAULT false, has_levels boolean NOT NULL DEFAULT false, - has_storage boolean NOT NULL DEFAULT false, has_invites boolean NOT NULL DEFAULT false, has_invite_achievements boolean NOT NULL DEFAULT false, - storage_config jsonb DEFAULT NULL, + storage jsonb DEFAULT NULL, + namespaces jsonb DEFAULT NULL, + functions jsonb DEFAULT NULL, + graphs jsonb DEFAULT NULL, + agents jsonb DEFAULT NULL, skip_entity_policies boolean NOT NULL DEFAULT false, table_provision jsonb DEFAULT NULL, out_membership_type int DEFAULT NULL, @@ -1547,6 +1550,16 @@ CREATE TABLE metaschema_modules_public.entity_type_provision ( out_files_table_id uuid DEFAULT NULL, out_path_shares_table_id uuid DEFAULT NULL, out_invites_module_id uuid DEFAULT NULL, + out_namespace_module_id uuid DEFAULT NULL, + out_namespaces_table_id uuid DEFAULT NULL, + out_namespace_events_table_id uuid DEFAULT NULL, + out_function_module_id uuid DEFAULT NULL, + out_definitions_table_id uuid DEFAULT NULL, + out_invocations_table_id uuid DEFAULT NULL, + out_execution_logs_table_id uuid DEFAULT NULL, + out_graph_module_id uuid DEFAULT NULL, + out_graphs_table_id uuid DEFAULT NULL, + out_agent_module_id uuid DEFAULT NULL, CONSTRAINT entity_type_provision_unique_prefix UNIQUE (database_id, prefix), CONSTRAINT entity_type_provision_db_fkey @@ -1608,18 +1621,13 @@ COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_levels IS Levels provide gamification/achievement tracking for members. When true, creates level steps, achievements, and level tables with security.'; -COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_storage IS 'Whether to provision storage_module for this type. Defaults to false. - When true, creates {prefix}_buckets and {prefix}_files tables - with entity-scoped RLS (AuthzEntityMembership) using the entity''s membership_type. - Storage tables get owner_id FK to the entity table, so files are owned by the entity.'; - COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_invites IS 'Whether to provision invites_module for this type. Defaults to false. When true, the trigger inserts a row into invites_module which in turn (via insert_invites_module BEFORE INSERT) creates {prefix}_invites and {prefix}_claimed_invites tables plus the submit_{prefix}_invite_code() function. - Symmetric counterpart of has_storage. Re-provisioning is idempotent: the - UNIQUE (database_id, membership_type) constraint on invites_module combined with - ON CONFLICT DO NOTHING in the fan-out makes repeated INSERTs safe.'; + Re-provisioning is idempotent: the UNIQUE (database_id, membership_type) constraint + on invites_module combined with ON CONFLICT DO NOTHING in the fan-out makes + repeated INSERTs safe.'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.has_invite_achievements IS 'Whether to auto-attach an EventTracker to the claimed_invites table for invite-based achievements. Defaults to false. Requires has_invites=true AND has_levels=true. @@ -1682,10 +1690,11 @@ COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_entity_tab COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_installed_modules IS 'Output: array of installed module labels (e.g. ARRAY[''permissions_module:data_room'', ''memberships_module:data_room'', ''invites_module:data_room'']). Populated by the trigger. Useful for verifying which modules were provisioned.'; -COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.storage_config IS 'Optional JSON array of storage module definitions. Each element provisions a separate - storage module with its own tables ({prefix}_{storage_key}_buckets/files), RLS policies, - and feature flags. Only used when has_storage = true; ignored otherwise. - NULL = provision a single default storage module with all defaults. +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.storage IS 'Optional JSON array of storage module definitions. Presence triggers provisioning + (same inference model as namespaces, functions, agents). + Each element provisions a separate storage module with its own tables + ({prefix}_{storage_key}_buckets/files), RLS policies, and feature flags. + NULL = do not provision storage. ''[{}]'' = provision one default storage module. Each array element recognizes (all optional): - storage_key (text) module discriminator, max 16 chars, lowercase snake_case. Defaults to ''default'' (omitted from table names). @@ -1707,20 +1716,60 @@ COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.storage_config - provisions (jsonb object) per-table customization keyed by "files" or "buckets". Each value: { nodes, fields, grants, use_rls, policies }. Example (single module, backward compat): - storage_config := ''[{"buckets": [{"name": "documents"}]}]''::jsonb + storage := ''[{"buckets": [{"name": "documents"}]}]''::jsonb Example (multi-module): - storage_config := ''[{"has_path_shares": true, "buckets": [{"name": "documents"}]}, {"storage_key": "fn", "has_custom_keys": true, "buckets": [{"name": "functions"}]}]''::jsonb'; + storage := ''[{"has_path_shares": true, "buckets": [{"name": "documents"}]}, {"storage_key": "fn", "has_custom_keys": true, "buckets": [{"name": "functions"}]}]''::jsonb'; -COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_storage_module_id IS 'Output: the UUID of the storage_module row created for this entity type. Populated by the trigger when has_storage=true.'; +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_storage_module_id IS 'Output: the UUID of the storage_module row created for this entity type. Populated by the trigger when storage is non-NULL and non-empty.'; -COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_buckets_table_id IS 'Output: the UUID of the generated buckets table (e.g. data_room_buckets). Populated by the trigger when has_storage=true.'; +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_buckets_table_id IS 'Output: the UUID of the generated buckets table (e.g. data_room_buckets). Populated by the trigger when storage is non-NULL and non-empty.'; -COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_files_table_id IS 'Output: the UUID of the generated files table (e.g. data_room_files). Populated by the trigger when has_storage=true.'; +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_files_table_id IS 'Output: the UUID of the generated files table (e.g. data_room_files). Populated by the trigger when storage is non-NULL and non-empty.'; COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_invites_module_id IS 'Output: the UUID of the invites_module row created for this entity type. Populated by the trigger when has_invites=true. NULL when has_invites=false, or when re-provisioning hits ON CONFLICT DO NOTHING (i.e. the invites_module row was created in a previous run).'; +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.namespaces IS 'Optional JSON array of namespace module definitions. Presence triggers provisioning. + NULL = do not provision namespaces. ''[{}]'' = provision one default namespace module. + Each element recognizes (all optional): + - key (text) module discriminator. Defaults to ''default''. + - policies (jsonb array) RLS policy overrides. NULL = apply defaults from apply_namespace_security(). + Creates {prefix}_namespaces (or {prefix}_{key}_namespaces for non-default keys) + with entity-scoped RLS (AuthzEntityMembership) and a rename proxy trigger. + Registers manage_namespaces permission bit on first provision. + Example: namespaces := ''[{}]''::jsonb'; + +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.functions IS 'Optional JSON array of function module definitions. Presence triggers provisioning. + NULL = do not provision functions. ''[{}]'' = provision one default function module. + Each element recognizes (all optional): + - key (text) module discriminator. Defaults to ''default''. + - policies (jsonb array) RLS policy overrides. NULL = apply defaults from apply_function_security(). + Creates {prefix}_function_definitions (or {prefix}_{key}_function_definitions for non-default keys) + with entity-scoped RLS and a job trigger dispatching function:provision tasks. + Registers manage_functions + invoke_functions permission bits on first provision. + Example: functions := ''[{}]''::jsonb'; + +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.graphs IS 'Optional JSON array of graph module definitions. Presence triggers provisioning. + NULL = do not provision graphs. ''[{}]'' = provision one default graph module. + Each element recognizes (all optional): + - key (text) module discriminator. Defaults to ''default''. + - policies (jsonb array) RLS policy overrides. NULL = apply defaults from apply_graph_security(). + Registers manage_graphs + execute_graphs permission bits on first provision. + Graph module requires a merkle_store_module_id dependency, so entity_type_provision + only registers permissions here. The graph module itself must be provisioned + separately with the merkle store dependency resolved. + Example: graphs := ''[{}]''::jsonb'; + +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_namespace_module_id IS 'Output: the UUID of the namespace_module row created (or found) for this entity type. + Populated by the trigger when namespaces is non-NULL. NULL otherwise.'; + +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_namespaces_table_id IS 'Output: the UUID of the generated namespaces table (e.g. data_room_namespaces). + Populated by the trigger when namespaces is non-NULL. NULL otherwise.'; + +COMMENT ON COLUMN metaschema_modules_public.entity_type_provision.out_namespace_events_table_id IS 'Output: the UUID of the generated namespace_events partitioned table (e.g. data_room_namespace_events). + Monthly partitioned, 12-month retention. Populated by the trigger when namespaces is non-NULL. NULL otherwise.'; + CREATE TABLE metaschema_modules_public.rate_limits_module ( id uuid PRIMARY KEY DEFAULT uuidv7(), database_id uuid NOT NULL, @@ -2118,6 +2167,8 @@ CREATE TABLE metaschema_modules_public.billing_module ( balances_table_name text NOT NULL DEFAULT '', meter_credits_table_id uuid NOT NULL DEFAULT uuid_nil(), meter_credits_table_name text NOT NULL DEFAULT '', + meter_sources_table_id uuid NOT NULL DEFAULT uuid_nil(), + meter_sources_table_name text NOT NULL DEFAULT '', record_usage_function text NOT NULL DEFAULT '', prefix text NULL, CONSTRAINT db_fkey @@ -2152,6 +2203,10 @@ CREATE TABLE metaschema_modules_public.billing_module ( FOREIGN KEY(meter_credits_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE, + CONSTRAINT meter_sources_table_fkey + FOREIGN KEY(meter_sources_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, CONSTRAINT billing_module_database_id_unique UNIQUE (database_id) ); @@ -2363,4 +2418,490 @@ CREATE INDEX config_secrets_org_module_schema_id_idx ON metaschema_modules_publi CREATE INDEX config_secrets_org_module_table_id_idx ON metaschema_modules_public.config_secrets_org_module (table_id); -COMMENT ON TABLE metaschema_modules_public.config_secrets_org_module IS 'Config row for the config_secrets_org_module, which provisions an organization-scoped encrypted key-value secrets store with manage_secrets permission and entity-membership RLS.'; \ No newline at end of file +COMMENT ON TABLE metaschema_modules_public.config_secrets_org_module IS 'Config row for the config_secrets_org_module, which provisions an organization-scoped encrypted key-value secrets store with manage_secrets permission and entity-membership RLS.'; + +CREATE TABLE metaschema_modules_public.inference_log_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + inference_log_table_id uuid NOT NULL DEFAULT uuid_nil(), + inference_log_table_name text NOT NULL DEFAULT '', + usage_daily_table_id uuid NOT NULL DEFAULT uuid_nil(), + usage_daily_table_name text NOT NULL DEFAULT '', + "interval" text NOT NULL DEFAULT '1 month', + retention text NOT NULL DEFAULT '12 months', + premake int NOT NULL DEFAULT 2, + scope text NOT NULL DEFAULT 'app', + actor_fk_table_id uuid NULL, + entity_fk_table_id uuid NULL, + prefix text NULL, + CONSTRAINT db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT inference_log_table_fkey + FOREIGN KEY(inference_log_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT usage_daily_table_fkey + FOREIGN KEY(usage_daily_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT inference_log_module_database_id_prefix_unique + UNIQUE NULLS NOT DISTINCT (database_id, prefix) +); + +CREATE INDEX inference_log_module_database_id_idx ON metaschema_modules_public.inference_log_module (database_id); + +CREATE TABLE metaschema_modules_public.compute_log_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + compute_log_table_id uuid NOT NULL DEFAULT uuid_nil(), + compute_log_table_name text NOT NULL DEFAULT '', + usage_daily_table_id uuid NOT NULL DEFAULT uuid_nil(), + usage_daily_table_name text NOT NULL DEFAULT '', + "interval" text NOT NULL DEFAULT '1 month', + retention text NOT NULL DEFAULT '12 months', + premake int NOT NULL DEFAULT 2, + scope text NOT NULL DEFAULT 'app', + actor_fk_table_id uuid NULL, + entity_fk_table_id uuid NULL, + prefix text NULL, + CONSTRAINT db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT compute_log_table_fkey + FOREIGN KEY(compute_log_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT usage_daily_table_fkey + FOREIGN KEY(usage_daily_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT compute_log_module_database_id_prefix_unique + UNIQUE NULLS NOT DISTINCT (database_id, prefix) +); + +CREATE INDEX compute_log_module_database_id_idx ON metaschema_modules_public.compute_log_module (database_id); + +CREATE TABLE metaschema_modules_public.transfer_log_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + transfer_log_table_id uuid NOT NULL DEFAULT uuid_nil(), + transfer_log_table_name text NOT NULL DEFAULT '', + usage_daily_table_id uuid NOT NULL DEFAULT uuid_nil(), + usage_daily_table_name text NOT NULL DEFAULT '', + "interval" text NOT NULL DEFAULT '1 month', + retention text NOT NULL DEFAULT '12 months', + premake int NOT NULL DEFAULT 2, + scope text NOT NULL DEFAULT 'app', + actor_fk_table_id uuid NULL, + entity_fk_table_id uuid NULL, + prefix text NULL, + CONSTRAINT db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT transfer_log_table_fkey + FOREIGN KEY(transfer_log_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT usage_daily_table_fkey + FOREIGN KEY(usage_daily_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT transfer_log_module_database_id_prefix_unique + UNIQUE NULLS NOT DISTINCT (database_id, prefix) +); + +CREATE INDEX transfer_log_module_database_id_idx ON metaschema_modules_public.transfer_log_module (database_id); + +CREATE TABLE metaschema_modules_public.storage_log_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + storage_log_table_id uuid NOT NULL DEFAULT uuid_nil(), + storage_log_table_name text NOT NULL DEFAULT '', + usage_daily_table_id uuid NOT NULL DEFAULT uuid_nil(), + usage_daily_table_name text NOT NULL DEFAULT '', + "interval" text NOT NULL DEFAULT '1 month', + retention text NOT NULL DEFAULT '12 months', + premake int NOT NULL DEFAULT 2, + scope text NOT NULL DEFAULT 'app', + actor_fk_table_id uuid NULL, + entity_fk_table_id uuid NULL, + prefix text NULL, + CONSTRAINT db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT storage_log_table_fkey + FOREIGN KEY(storage_log_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT usage_daily_table_fkey + FOREIGN KEY(usage_daily_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT storage_log_module_database_id_prefix_unique + UNIQUE NULLS NOT DISTINCT (database_id, prefix) +); + +CREATE INDEX storage_log_module_database_id_idx ON metaschema_modules_public.storage_log_module (database_id); + +CREATE TABLE metaschema_modules_public.db_usage_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + table_stats_log_table_id uuid NOT NULL DEFAULT uuid_nil(), + table_stats_log_table_name text NOT NULL DEFAULT '', + table_stats_daily_table_id uuid NOT NULL DEFAULT uuid_nil(), + table_stats_daily_table_name text NOT NULL DEFAULT '', + query_stats_log_table_id uuid NOT NULL DEFAULT uuid_nil(), + query_stats_log_table_name text NOT NULL DEFAULT '', + query_stats_daily_table_id uuid NOT NULL DEFAULT uuid_nil(), + query_stats_daily_table_name text NOT NULL DEFAULT '', + "interval" text NOT NULL DEFAULT '1 month', + retention text NOT NULL DEFAULT '12 months', + premake int NOT NULL DEFAULT 2, + scope text NOT NULL DEFAULT 'app', + prefix text NULL, + CONSTRAINT db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT table_stats_log_table_fkey + FOREIGN KEY(table_stats_log_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT table_stats_daily_table_fkey + FOREIGN KEY(table_stats_daily_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT query_stats_log_table_fkey + FOREIGN KEY(query_stats_log_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT query_stats_daily_table_fkey + FOREIGN KEY(query_stats_daily_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT db_usage_module_database_id_prefix_unique + UNIQUE NULLS NOT DISTINCT (database_id, prefix) +); + +CREATE INDEX db_usage_module_database_id_idx ON metaschema_modules_public.db_usage_module (database_id); + +CREATE TABLE metaschema_modules_public.agent_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + thread_table_id uuid NOT NULL DEFAULT uuid_nil(), + message_table_id uuid NOT NULL DEFAULT uuid_nil(), + task_table_id uuid NOT NULL DEFAULT uuid_nil(), + prompts_table_id uuid NOT NULL DEFAULT uuid_nil(), + knowledge_table_id uuid DEFAULT NULL, + thread_table_name text NOT NULL DEFAULT 'agent_thread', + message_table_name text NOT NULL DEFAULT 'agent_message', + task_table_name text NOT NULL DEFAULT 'agent_task', + prompts_table_name text NOT NULL DEFAULT 'agent_prompt', + knowledge_table_name text NOT NULL DEFAULT 'agent_knowledge', + has_knowledge boolean NOT NULL DEFAULT false, + api_name text DEFAULT 'agent', + membership_type int DEFAULT NULL, + entity_table_id uuid NULL, + policies jsonb NULL, + knowledge_config jsonb NULL, + knowledge_policies jsonb NULL, + provisions jsonb NULL, + CONSTRAINT agent_module_db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_thread_table_fkey + FOREIGN KEY(thread_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_message_table_fkey + FOREIGN KEY(message_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_task_table_fkey + FOREIGN KEY(task_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_prompts_table_fkey + FOREIGN KEY(prompts_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_knowledge_table_fkey + FOREIGN KEY(knowledge_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT agent_module_entity_table_fkey + FOREIGN KEY(entity_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE +); + +CREATE INDEX agent_module_database_id_idx ON metaschema_modules_public.agent_module (database_id); + +CREATE UNIQUE INDEX agent_module_unique_scope ON metaschema_modules_public.agent_module (database_id, (COALESCE(membership_type, -1))); + +CREATE TABLE metaschema_modules_public.merkle_store_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + public_schema_name text, + object_table_id uuid NOT NULL DEFAULT uuid_nil(), + store_table_id uuid NOT NULL DEFAULT uuid_nil(), + commit_table_id uuid NOT NULL DEFAULT uuid_nil(), + ref_table_id uuid NOT NULL DEFAULT uuid_nil(), + prefix text NOT NULL DEFAULT '', + api_name text, + scope_field text NOT NULL DEFAULT 'scope_id', + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT object_table_fkey + FOREIGN KEY(object_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT store_table_fkey + FOREIGN KEY(store_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT commit_table_fkey + FOREIGN KEY(commit_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT ref_table_fkey + FOREIGN KEY(ref_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT merkle_store_module_database_prefix_unique + UNIQUE (database_id, prefix) +); + +CREATE INDEX merkle_store_module_database_id_idx ON metaschema_modules_public.merkle_store_module (database_id); + +CREATE TABLE metaschema_modules_public.graph_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + public_schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + public_schema_name text, + private_schema_name text, + prefix text NOT NULL DEFAULT '', + merkle_store_module_id uuid NOT NULL, + graphs_table_id uuid NOT NULL DEFAULT uuid_nil(), + executions_table_id uuid NOT NULL DEFAULT uuid_nil(), + outputs_table_id uuid NOT NULL DEFAULT uuid_nil(), + api_name text, + private_api_name text, + scope_field text NOT NULL DEFAULT 'scope_id', + membership_type int DEFAULT NULL, + entity_table_id uuid NULL, + policies jsonb NULL, + provisions jsonb NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT public_schema_fkey + FOREIGN KEY(public_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT merkle_store_fkey + FOREIGN KEY(merkle_store_module_id) + REFERENCES metaschema_modules_public.merkle_store_module (id) + ON DELETE CASCADE, + CONSTRAINT graphs_table_fkey + FOREIGN KEY(graphs_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT executions_table_fkey + FOREIGN KEY(executions_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT outputs_table_fkey + FOREIGN KEY(outputs_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT graph_module_entity_table_fkey + FOREIGN KEY(entity_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT graph_module_database_merkle_unique + UNIQUE (database_id, merkle_store_module_id) +); + +CREATE INDEX graph_module_database_id_idx ON metaschema_modules_public.graph_module (database_id); + +CREATE TABLE metaschema_modules_public.namespace_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + public_schema_name text, + private_schema_name text, + namespaces_table_id uuid NOT NULL DEFAULT uuid_nil(), + namespace_events_table_id uuid NOT NULL DEFAULT uuid_nil(), + namespaces_table_name text NOT NULL DEFAULT 'namespaces', + namespace_events_table_name text NOT NULL DEFAULT 'namespace_events', + api_name text, + private_api_name text, + membership_type int DEFAULT NULL, + entity_table_id uuid NULL, + policies jsonb NULL, + provisions jsonb NULL, + CONSTRAINT namespace_module_db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT namespace_module_schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT namespace_module_private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT namespace_module_namespaces_table_fkey + FOREIGN KEY(namespaces_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT namespace_module_events_table_fkey + FOREIGN KEY(namespace_events_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT namespace_module_entity_table_fkey + FOREIGN KEY(entity_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE +); + +CREATE INDEX namespace_module_database_id_idx ON metaschema_modules_public.namespace_module (database_id); + +CREATE UNIQUE INDEX namespace_module_unique_scope ON metaschema_modules_public.namespace_module (database_id, (COALESCE(membership_type, -1))); + +CREATE TABLE metaschema_modules_public.function_module ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + public_schema_name text, + private_schema_name text, + definitions_table_id uuid NOT NULL DEFAULT uuid_nil(), + invocations_table_id uuid NOT NULL DEFAULT uuid_nil(), + execution_logs_table_id uuid NOT NULL DEFAULT uuid_nil(), + definitions_table_name text NOT NULL DEFAULT 'function_definitions', + invocations_table_name text NOT NULL DEFAULT 'function_invocations', + execution_logs_table_name text NOT NULL DEFAULT 'function_execution_logs', + api_name text, + private_api_name text, + membership_type int DEFAULT NULL, + entity_table_id uuid NULL, + policies jsonb NULL, + provisions jsonb NULL, + CONSTRAINT function_module_db_fkey + FOREIGN KEY(database_id) + REFERENCES metaschema_public.database (id) + ON DELETE CASCADE, + CONSTRAINT function_module_schema_fkey + FOREIGN KEY(schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT function_module_private_schema_fkey + FOREIGN KEY(private_schema_id) + REFERENCES metaschema_public.schema (id) + ON DELETE CASCADE, + CONSTRAINT function_module_definitions_table_fkey + FOREIGN KEY(definitions_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT function_module_invocations_table_fkey + FOREIGN KEY(invocations_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT function_module_execution_logs_table_fkey + FOREIGN KEY(execution_logs_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE, + CONSTRAINT function_module_entity_table_fkey + FOREIGN KEY(entity_table_id) + REFERENCES metaschema_public.table (id) + ON DELETE CASCADE +); + +CREATE INDEX function_module_database_id_idx ON metaschema_modules_public.function_module (database_id); + +CREATE UNIQUE INDEX function_module_unique_scope ON metaschema_modules_public.function_module (database_id, (COALESCE(membership_type, -1))); \ No newline at end of file diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/billing_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/billing_module/table.sql index 3282037d..aa1c1c91 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/billing_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/billing_module/table.sql @@ -15,6 +15,8 @@ SELECT ledger_table_name, balances_table_id, balances_table_name, + meter_sources_table_id, + meter_sources_table_name, record_usage_function, prefix FROM metaschema_modules_public.billing_module diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/compute_log_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/compute_log_module/table.sql index fbdce50f..de44722a 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/compute_log_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/compute_log_module/table.sql @@ -3,8 +3,7 @@ SELECT id, database_id, schema_id, private_schema_id, compute_log_table_id, compute_log_table_name, usage_daily_table_id, usage_daily_table_name, - "interval", retention, premake, scope, - actor_fk_table_id, entity_fk_table_id, + retention, scope, actor_fk_table_id, entity_fk_table_id, prefix FROM metaschema_modules_public.compute_log_module WHERE FALSE; diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/db_usage_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/db_usage_module/table.sql index b7ec9d21..57f884fe 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/db_usage_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/db_usage_module/table.sql @@ -5,7 +5,6 @@ SELECT id, database_id, schema_id, private_schema_id, table_stats_daily_table_id, table_stats_daily_table_name, query_stats_log_table_id, query_stats_log_table_name, query_stats_daily_table_id, query_stats_daily_table_name, - "interval", retention, premake, scope, - prefix + retention, scope, prefix FROM metaschema_modules_public.db_usage_module WHERE FALSE; diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/inference_log_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/inference_log_module/table.sql index a5c1c2aa..eaf4c39c 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/inference_log_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/inference_log_module/table.sql @@ -11,9 +11,10 @@ SELECT inference_log_table_name, usage_daily_table_id, usage_daily_table_name, - "interval", retention, - premake, + scope, + actor_fk_table_id, + entity_fk_table_id, prefix FROM metaschema_modules_public.inference_log_module WHERE FALSE; diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/namespace_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/namespace_module/table.sql index da2fdc7e..ee614746 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/namespace_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/namespace_module/table.sql @@ -4,9 +4,10 @@ BEGIN; SELECT id, database_id, schema_id, private_schema_id, public_schema_name, private_schema_name, - namespaces_table_id, namespaces_table_name, + namespaces_table_id, namespace_events_table_id, + namespaces_table_name, namespace_events_table_name, api_name, private_api_name, - membership_type, entity_table_id, policies + entity_table_id, policies, provisions FROM metaschema_modules_public.namespace_module WHERE false; diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/profiles_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/profiles_module/table.sql index 64810749..9c52869f 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/profiles_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/profiles_module/table.sql @@ -6,7 +6,8 @@ SELECT id, database_id, schema_id, private_schema_id, table_id, table_name, profile_permissions_table_id, profile_permissions_table_name, profile_grants_table_id, profile_grants_table_name, profile_definition_grants_table_id, profile_definition_grants_table_name, - membership_type, entity_table_id, actor_table_id, + profile_templates_table_id, profile_templates_table_name, + entity_table_id, actor_table_id, permissions_table_id, memberships_table_id, prefix FROM metaschema_modules_public.profiles_module WHERE FALSE; diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/storage_log_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/storage_log_module/table.sql index 3ef40c68..adbf9ac7 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/storage_log_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/storage_log_module/table.sql @@ -3,7 +3,7 @@ SELECT id, database_id, schema_id, private_schema_id, storage_log_table_id, storage_log_table_name, usage_daily_table_id, usage_daily_table_name, - "interval", retention, premake, scope, + retention, scope, actor_fk_table_id, entity_fk_table_id, prefix FROM metaschema_modules_public.storage_log_module diff --git a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/transfer_log_module/table.sql b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/transfer_log_module/table.sql index 58758b2f..6a17ba16 100644 --- a/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/transfer_log_module/table.sql +++ b/packages/metaschema-modules/verify/schemas/metaschema_modules_public/tables/transfer_log_module/table.sql @@ -3,7 +3,7 @@ SELECT id, database_id, schema_id, private_schema_id, transfer_log_table_id, transfer_log_table_name, usage_daily_table_id, usage_daily_table_name, - "interval", retention, premake, scope, + retention, scope, actor_fk_table_id, entity_fk_table_id, prefix FROM metaschema_modules_public.transfer_log_module diff --git a/packages/metaschema-schema/deploy/schemas/metaschema_public/tables/table_grant/table.sql b/packages/metaschema-schema/deploy/schemas/metaschema_public/tables/table_grant/table.sql index 68273ad2..9b4ccc4b 100644 --- a/packages/metaschema-schema/deploy/schemas/metaschema_public/tables/table_grant/table.sql +++ b/packages/metaschema-schema/deploy/schemas/metaschema_public/tables/table_grant/table.sql @@ -26,4 +26,11 @@ CREATE TABLE metaschema_public.table_grant ( CREATE INDEX table_grant_table_id_idx ON metaschema_public.table_grant ( table_id ); CREATE INDEX table_grant_database_id_idx ON metaschema_public.table_grant ( database_id ); +CREATE UNIQUE INDEX table_grant_unique_idx ON metaschema_public.table_grant ( + table_id, + privilege, + grantee_name, + COALESCE(field_ids, '{}'::uuid[]) +); + COMMIT; diff --git a/packages/metaschema-schema/sql/metaschema-schema--0.26.0.sql b/packages/metaschema-schema/sql/metaschema-schema--0.26.0.sql index f1ad26b8..c5781558 100644 --- a/packages/metaschema-schema/sql/metaschema-schema--0.26.0.sql +++ b/packages/metaschema-schema/sql/metaschema-schema--0.26.0.sql @@ -388,6 +388,8 @@ CREATE INDEX table_grant_table_id_idx ON metaschema_public.table_grant (table_id CREATE INDEX table_grant_database_id_idx ON metaschema_public.table_grant (database_id); +CREATE UNIQUE INDEX table_grant_unique_idx ON metaschema_public.table_grant (table_id, privilege, grantee_name, (COALESCE(field_ids, CAST('{}' AS uuid[])))); + CREATE FUNCTION metaschema_private.table_name_hash( name text ) RETURNS bytea AS $EOFCODE$ diff --git a/packages/object-store/README.md b/packages/object-store/README.md index 2b46009f..1d882f62 100644 --- a/packages/object-store/README.md +++ b/packages/object-store/README.md @@ -12,28 +12,19 @@

-A content-addressable, immutable object store for PostgreSQL — Merkle trees, all the way down. Pair with [`@pgpm/object-tree`](https://www.npmjs.com/package/@pgpm/object-tree) for Git-like version control. +Immutable versioned object storage with content-addressable IDs for PostgreSQL ## Overview -Born from the conviction that the best ideas in computer science keep getting reinvented because they keep being right, `@pgpm/object-store` brings content-addressable storage and structural sharing into PostgreSQL. Inspired by Rich Hickey's philosophy that immutability is not a constraint but a liberation — that the past should never be silently rewritten — this package implements a Merkle tree directly in SQL, where every object's identity is derived from what it contains rather than where it sits. - -The original code was written in Egypt, during a period of deep focus on what it would mean to treat a relational database the way Hickey's Datomic treats time: as an accretion of immutable facts rather than a place that forgets. The result is a storage engine where objects are permanent, identity is derived from content, and every version of every tree is always available — not because you remembered to snapshot it, but because the data structure itself makes forgetting impossible. - -The design follows a simple principle borrowed from Git's object model: if you know the content, you know the identity. Objects are stored as nodes in a tree. Each node holds a JSON payload and an ordered list of named children. A trigger hashes the content on every insert, producing a deterministic UUID. Two objects with identical data and children always produce the same ID — deduplication and integrity verification happen automatically, by construction, not by convention. - -Once an object is frozen, it becomes truly immutable — no updates, no deletes, enforced by triggers at the database level. New versions don't destroy old ones; they share structure with them. Change one leaf and only the nodes along the path from that leaf to the root are recreated. Everything else is reused. This is structural sharing — the same trick that makes persistent data structures in Clojure and Haskell efficient — running inside your database. +`@pgpm/object-store` implements a content-addressable Merkle tree storage system in PostgreSQL. Objects are immutable once frozen, and their IDs are deterministically derived from their content (data + children), similar to how Git stores objects. This enables efficient structural sharing, deduplication, and tamper-evident storage. ## Features -- **Content-Addressable IDs**: Object IDs are deterministically computed from content using UUID v5 hashing — same content always produces the same ID -- **Merkle Tree Structure**: Objects form a tree via parallel `kids`/`ktree` arrays, enabling structural sharing across versions -- **Immutability Enforcement**: Frozen objects cannot be modified or deleted, enforced at the trigger level -- **Path-Based Operations**: Insert, update, remove, and query nodes using hierarchical paths like a filesystem -- **Structural Sharing**: Unchanged subtrees are reused across versions (copy-on-write), keeping storage efficient -- **Recursive Traversal**: Walk entire trees or paths from root to leaf in a single query -- **Scope Isolation**: Multi-tenant by design — `scope_id` partitions all data without foreign keys -- **Array Utilities**: Built-in helper functions for array manipulation (`array_shift`, `array_pop`, `array_index_of`, etc.) +- **Content-Addressable IDs**: Object IDs are deterministically computed from content using UUID v5 hashing +- **Merkle Tree Structure**: Objects form a tree via `kids`/`ktree` arrays, enabling structural sharing +- **Immutability**: Frozen objects cannot be modified or deleted +- **Path-Based Operations**: Insert, update, remove, and query nodes using hierarchical paths +- **Structural Sharing**: Unchanged subtrees are reused across versions (copy-on-write) - **Pure plpgsql**: No external dependencies beyond pgcrypto and uuid-ossp ## Installation @@ -90,381 +81,96 @@ pgpm deploy --createdb --database mydb1 ## Core Concepts -### The Object Table - -Every object in the store is a row in `object_store_public.object`: +### Object Table -| Column | Type | Description | -|--------|------|-------------| -| `id` | uuid | Content-addressable hash, computed automatically from `data` + `kids`/`ktree` | -| `scope_id` | uuid | Tenant/namespace isolation key | -| `data` | jsonb | The object's payload — any JSON value | -| `kids` | uuid[] | Ordered array of child object IDs | -| `ktree` | text[] | Ordered array of child names (parallel to `kids`) | -| `frzn` | bool | Whether the object is frozen (immutable) | -| `created_at` | timestamptz | When the object was first stored | - -The primary key is `(id, scope_id)`. The `kids` and `ktree` arrays must always have the same length (enforced by a CHECK constraint), or both be NULL for leaf nodes. +Every object has: +- `id` (uuid) — content-addressable hash computed from data + children +- `scope_id` (uuid) — scoping identifier for multi-tenant usage +- `data` (jsonb) — the object's payload +- `kids` (uuid[]) — child object IDs +- `ktree` (text[]) — child names (parallel to `kids`) +- `frzn` (bool) — whether the object is frozen (immutable) ### Content-Addressable Hashing -Every time you insert an object, a `BEFORE INSERT` trigger computes its ID by hashing the `data` and `kids`/`ktree` arrays using UUID v5. This means: - -- Two objects with identical content always have the same ID -- Deduplication is automatic — inserting the same content twice is a no-op -- Any change to content produces a completely different ID -- You can verify integrity by recomputing the hash - -This is the same principle behind Git's object store and Merkle trees in general. - -### Structural Sharing - -When you update a single node deep in a tree, only the nodes along the path from that node to the root are recreated. All other subtrees are shared by reference: - -``` -Before: After updating C: - - R R' - / \ / \ - A B --> A B' - / \ \ / \ \ - C D E C' D E -``` - -Only R, B, and C are new objects. A, D, and E are reused. This keeps the storage cost proportional to the depth of the change, not the size of the tree. - -### Immutability and Freezing - -Objects start mutable (`frzn = false`). Once frozen via `freeze_objects()`, they become permanently immutable: - -- **Updates are blocked**: Any attempt to change a frozen object's `id`, `data`, `kids`, or `ktree` raises an exception -- **Deletes are blocked**: Frozen objects cannot be deleted -- **Freezing is recursive**: `freeze_objects()` freezes the target and all its descendants -- **The only allowed transition** is `frzn: false -> true` (you can freeze an unfrozen object, but never unfreeze) - -This gives you an append-only history where past states are preserved by construction. +Object IDs are automatically generated via a trigger that hashes the object's `data` and `kids`/`ktree` arrays. Two objects with identical content will always have the same ID, enabling natural deduplication and structural sharing. ## Core Functions -### object_store_public.insert_node_at_path(s_id, root, path, data, kids, ktree) - -Insert or replace a node at the given path, creating intermediate nodes as needed. Returns the new root ID (since all ancestors are recreated with updated children via structural sharing). +### object_store_public.insert_node_at_path(scope_id, root, path, data, kids, ktree) -**Signature:** -```sql -object_store_public.insert_node_at_path( - s_id uuid, -- scope identifier - root uuid, -- current root object ID - path text[], -- path to the target node - data jsonb, -- JSON payload for the new node - kids uuid[], -- child object IDs (empty array for leaf nodes) - ktree text[] -- child names (empty array for leaf nodes) -) RETURNS uuid -- new root ID -``` +Insert or replace a node at the given path, creating intermediate nodes as needed. Returns the new root ID (since all ancestors are recreated with updated children). -**Usage:** ```sql --- Create a root node -INSERT INTO object_store_public.object (scope_id) - VALUES ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa') - RETURNING id; --- Returns: - --- Insert a leaf node at path ['src', 'main.ts'] SELECT object_store_public.insert_node_at_path( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid, - ARRAY['src', 'main.ts'], - '{"content": "console.log(hello)"}'::jsonb, - ARRAY[]::uuid[], - ARRAY[]::text[] -); --- Returns: -``` - -### object_store_public.set_data_at_path(s_id, root, path, data) - -Update the data on an existing node while preserving its children. Looks up the existing node's `kids`/`ktree` first, then delegates to `insert_node_at_path`. - -**Signature:** -```sql -object_store_public.set_data_at_path( - s_id uuid, -- scope identifier - root uuid, -- current root object ID - path text[], -- path to the target node - data jsonb -- new JSON payload -) RETURNS uuid -- new root ID -``` - -**Usage:** -```sql --- Update only the data at a path, keeping children intact -SELECT object_store_public.set_data_at_path( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid, - ARRAY['src', 'main.ts'], - '{"content": "console.log(updated)"}'::jsonb + s_id := 'aaaaaaaa-...'::uuid, + root := ''::uuid, + path := ARRAY['src', 'main.ts']::text[], + data := '{"content": "console.log(hello)"}'::jsonb, + kids := ARRAY[]::uuid[], + ktree := ARRAY[]::text[] ); ``` -### object_store_public.get_node_at_path(s_id, id, path) +### object_store_public.get_node_at_path(scope_id, id, path) -Retrieve the object at a given path from a root node. Returns the full object row. +Retrieve the object at a given path from a root node. -**Signature:** ```sql -object_store_public.get_node_at_path( - s_id uuid, -- scope identifier - id uuid, -- root object ID - path text[] -- path to traverse (empty array returns the root itself) -) RETURNS object_store_public.object -``` - -**Usage:** -```sql --- Get the object at ['src', 'main.ts'] -SELECT * FROM object_store_public.get_node_at_path( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid, - ARRAY['src', 'main.ts'] -); - --- Get the root object itself SELECT * FROM object_store_public.get_node_at_path( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid, - ARRAY[]::text[] + s_id := 'aaaaaaaa-...'::uuid, + id := ''::uuid, + path := ARRAY['src', 'main.ts']::text[] ); ``` -### object_store_public.remove_node_at_path(s_id, root, path) +### object_store_public.remove_node_at_path(scope_id, root, path) -Remove a node at the given path. Returns a new root with the node removed from its parent's children. If the node doesn't exist, returns the original root unchanged. Cannot remove the root node itself. +Remove a node at the given path, returning a new root with the node removed. -**Signature:** ```sql -object_store_public.remove_node_at_path( - s_id uuid, -- scope identifier - root uuid, -- current root object ID - path text[] -- path to the node to remove -) RETURNS uuid -- new root ID -``` - -**Usage:** -```sql --- Remove the node at ['src', 'old-file.ts'] SELECT object_store_public.remove_node_at_path( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid, - ARRAY['src', 'old-file.ts'] + s_id := 'aaaaaaaa-...'::uuid, + root := ''::uuid, + path := ARRAY['src', 'old-file.ts']::text[] ); ``` -### object_store_public.get_all_objects_from_root(s_id, id) +### object_store_public.set_data_at_path(scope_id, root, path, data) -Recursively retrieve all objects in the tree starting from a root node. Uses a recursive CTE to walk the entire `kids` graph. - -**Signature:** -```sql -object_store_public.get_all_objects_from_root( - s_id uuid, -- scope identifier - id uuid -- root object ID -) RETURNS SETOF object_store_public.object -``` +Update the data on an existing node while preserving its children. -**Usage:** ```sql --- Get every object in the tree -SELECT * FROM object_store_public.get_all_objects_from_root( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid -); -``` - -### object_store_public.get_path_objects_from_root(s_id, id, path) - -Retrieve all objects along a path from root to the target node. Returns one row per node along the path, starting with the root. - -**Signature:** -```sql -object_store_public.get_path_objects_from_root( - s_id uuid, -- scope identifier - id uuid, -- root object ID - path text[] -- path to walk -) RETURNS SETOF object_store_public.object -``` - -**Usage:** -```sql --- Get every object along the path ['src', 'components', 'Button.tsx'] -SELECT * FROM object_store_public.get_path_objects_from_root( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid, - ARRAY['src', 'components', 'Button.tsx'] +SELECT object_store_public.set_data_at_path( + s_id := 'aaaaaaaa-...'::uuid, + root := ''::uuid, + path := ARRAY['src', 'main.ts']::text[], + data := '{"content": "updated content"}'::jsonb ); --- Returns: root, src node, components node, Button.tsx node ``` -### object_store_public.get_all(s_id, id) +### object_store_public.get_all_objects_from_root(scope_id, id) -Recursively retrieve all paths and their data from a root node. Returns `(path, data)` tuples for every node in the tree, walking children depth-first. +Recursively retrieve all objects in the tree starting from a root node. -**Signature:** ```sql -object_store_public.get_all( - s_id uuid, -- scope identifier - id uuid -- root object ID -) RETURNS TABLE (path text[], data jsonb) -``` - -**Usage:** -```sql --- Flatten the tree into path/data pairs -SELECT * FROM object_store_public.get_all( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid +SELECT * FROM object_store_public.get_all_objects_from_root( + s_id := 'aaaaaaaa-...'::uuid, + id := ''::uuid ); --- Returns rows like: --- path: ['src', 'main.ts'], data: '{"content": "..."}' --- path: ['src', 'utils.ts'], data: '{"content": "..."}' --- path: ['README.md'], data: '{"content": "..."}' --- path: [], data: null (root node) ``` -### object_store_public.freeze_objects(s_id, id) - -Recursively freeze an object and all its descendants. Once frozen, objects are permanently immutable — enforced by database triggers. +### object_store_public.freeze_objects(scope_id, id) -**Signature:** -```sql -object_store_public.freeze_objects( - s_id uuid, -- scope identifier - id uuid -- root object ID to freeze -) RETURNS void -``` +Freeze an object and all its descendants, making them immutable. -**Usage:** ```sql --- Freeze the entire tree SELECT object_store_public.freeze_objects( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ''::uuid + s_id := 'aaaaaaaa-...'::uuid, + id := ''::uuid ); - --- Now any modification attempt will fail: --- UPDATE object_store_public.object SET data = '{}' WHERE id = ''; --- ERROR: you cannot mutate an immutable record. ``` -### object_store_public.update_node_at_path(s_id, root, path, data, kids, ktree) - -Replace an existing node at the given path entirely (data + children). Delegates to `insert_node_at_path`. - -**Signature:** -```sql -object_store_public.update_node_at_path( - s_id uuid, -- scope identifier - root uuid, -- current root object ID - path text[], -- path to the target node - data jsonb, -- new JSON payload - kids uuid[], -- new child object IDs - ktree text[] -- new child names -) RETURNS uuid -- new root ID -``` - -## Usage Examples - -### Building a Virtual Filesystem - -```sql -DO $$ -DECLARE - scope uuid := 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'; - root_id uuid; - new_root uuid; -BEGIN - -- Create an empty root - INSERT INTO object_store_public.object (scope_id) - VALUES (scope) RETURNING id INTO root_id; - - -- Add files - SELECT object_store_public.insert_node_at_path( - scope, root_id, ARRAY['README.md'], - '{"content": "# My Project"}'::jsonb, - ARRAY[]::uuid[], ARRAY[]::text[] - ) INTO new_root; - - SELECT object_store_public.insert_node_at_path( - scope, new_root, ARRAY['src', 'index.ts'], - '{"content": "export default {}"}'::jsonb, - ARRAY[]::uuid[], ARRAY[]::text[] - ) INTO new_root; - - -- List all files - PERFORM * FROM object_store_public.get_all(scope, new_root); -END $$; -``` - -### Comparing Versions - -```sql --- After making changes, you have two root IDs (old and new). --- Get all objects from each to diff them: - -SELECT 'added' as status, path, data -FROM object_store_public.get_all(scope, new_root) -WHERE path NOT IN (SELECT path FROM object_store_public.get_all(scope, old_root)) - -UNION ALL - -SELECT 'removed' as status, path, data -FROM object_store_public.get_all(scope, old_root) -WHERE path NOT IN (SELECT path FROM object_store_public.get_all(scope, new_root)); -``` - -### Multi-Tenant Isolation - -```sql --- Each tenant gets their own scope_id. --- Objects are completely isolated between scopes. - --- Tenant A -SELECT object_store_public.insert_node_at_path( - 'aaaaaaaa-0000-0000-0000-000000000001', root_a, ... -); - --- Tenant B (completely separate namespace) -SELECT object_store_public.insert_node_at_path( - 'aaaaaaaa-0000-0000-0000-000000000002', root_b, ... -); -``` - -## Database Schema - -### Schemas - -| Schema | Purpose | -|--------|---------| -| `object_store_public` | Public API — the object table and all user-facing functions | -| `object_store_private` | Internal — hash computation and trigger functions | -| `object_store_utils` | Array utility functions used by the store internals | - -### Indexes - -| Index | Table | Columns | Purpose | -|-------|-------|---------|---------| -| `scope_id_idx` | object | `scope_id` | Fast scope-based queries | -| `frzn_idx` | object | `frzn` | Efficient frozen/unfrozen filtering | -| `object_kids_idx` | object | `kids` (GIN) | Fast child lookups for recursive traversals | - -### Triggers - -| Trigger | Event | Purpose | -|---------|-------|---------| -| `generate_id_hash` | BEFORE INSERT | Computes content-addressable ID from data + children | -| `immutable_objects` | BEFORE UPDATE | Prevents modification of frozen objects | -| `delete_immutable_objects` | BEFORE DELETE | Prevents deletion of frozen objects | - ## Testing ```bash @@ -473,21 +179,12 @@ pnpm test ## Dependencies -- [`@pgpm/verify`](https://www.npmjs.com/package/@pgpm/verify): Verification utilities - -## See Also - -- [`@pgpm/object-tree`](https://www.npmjs.com/package/@pgpm/object-tree): Git-like version control layer built on top of this package — adds commits, refs, and stores for full history tracking +- `@pgpm/verify`: Verification utilities ## Related Tooling -* [pgpm](https://github.com/constructive-io/constructive/tree/main/packages/pgpm): **PostgreSQL Package Manager** for modular Postgres development. Works with database workspaces, scaffolding, migrations, seeding, and installing database packages. -* [pgsql-test](https://github.com/constructive-io/constructive/tree/main/packages/pgsql-test): **Isolated testing environments** with per-test transaction rollbacks — ideal for integration tests, complex migrations, and RLS simulation. -* [supabase-test](https://github.com/constructive-io/constructive/tree/main/packages/supabase-test): **Supabase-native test harness** preconfigured for the local Supabase stack — per-test rollbacks, JWT/role context helpers, and CI/GitHub Actions ready. -* [graphile-test](https://github.com/constructive-io/constructive/tree/main/packages/graphile-test): **Authentication mocking** for Graphile-focused test helpers and emulating row-level security contexts. -* [pgsql-parser](https://github.com/constructive-io/pgsql-parser): **SQL conversion engine** that interprets and converts PostgreSQL syntax. -* [libpg-query-node](https://github.com/constructive-io/libpg-query-node): **Node.js bindings** for `libpg_query`, converting SQL into parse trees. -* [pg-proto-parser](https://github.com/constructive-io/pg-proto-parser): **Protobuf parser** for parsing PostgreSQL Protocol Buffers definitions to generate TypeScript interfaces, utility functions, and JSON mappings for enums. +* [pgpm](https://github.com/constructive-io/constructive/tree/main/packages/pgpm): PostgreSQL Package Manager for modular Postgres development. +* [pgsql-test](https://github.com/constructive-io/constructive/tree/main/packages/pgsql-test): Isolated testing environments with per-test transaction rollbacks. ## Disclaimer diff --git a/packages/object-tree/Makefile b/packages/object-tree/Makefile index b3a61ccd..20ed0470 100644 --- a/packages/object-tree/Makefile +++ b/packages/object-tree/Makefile @@ -1,5 +1,5 @@ EXTENSION = object-tree -DATA = sql/object-tree--0.15.5.sql +DATA = sql/object-tree--0.26.0.sql PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/packages/object-tree/README.md b/packages/object-tree/README.md index 76392c86..ad0608d6 100644 --- a/packages/object-tree/README.md +++ b/packages/object-tree/README.md @@ -12,26 +12,22 @@

-Git-like version control for database objects, built on [`@pgpm/object-store`](https://www.npmjs.com/package/@pgpm/object-store)'s immutable Merkle trees. +Git-like version control for database objects with commits and refs ## Overview -If `@pgpm/object-store` is the storage engine, `@pgpm/object-tree` is the version control system built on top of it. Like Git, it layers commits, refs, and stores over a content-addressable object graph — but instead of tracking files on a filesystem, it tracks structured data inside PostgreSQL. - -This package was born alongside `@pgpm/object-store`, written in Egypt during the same stretch of deep work on immutable database architectures. The inspiration, again, was Rich Hickey and the ideas behind Datomic: that a database should remember everything, that time is a first-class concept, and that you should be able to ask "what did this look like yesterday?" as naturally as you ask "what does this look like now?" Where the object store provides the immutable foundation — content-addressable nodes, structural sharing, Merkle integrity — the object tree adds the narrative layer: commits that record *when* things changed, refs that name the current state, and stores that isolate independent histories from each other. - -The result is a version-controlled data layer that lives entirely inside your database. Every mutation is a commit. Every commit points to an immutable tree root. You can branch, diff, and time-travel through your data's history using the same mental model you use with Git — except the objects are rows, and the repository is a schema. +`@pgpm/object-tree` builds on top of `@pgpm/object-store` to provide a Git-like version control layer for database objects. It adds commits, refs (branches), and stores to organize content-addressable object trees into a full version history with time-travel capabilities. ## Features -- **Commits**: Track changes with parent references, messages, and timestamps — a full history chain -- **Refs / Branches**: Named pointers to commits (like Git branches), with `main` as the default -- **Stores**: Isolated repositories within a single database — multiple independent histories per scope -- **Rev-Parse**: Resolve a ref name to the current tree root in a single call -- **Set-and-Commit**: Atomic insert-and-commit in a single operation — no partial states -- **Set-Props-and-Commit**: Update node properties (preserving children) and commit atomically -- **Time Travel**: Access any historical state via commit IDs — every past tree root is still available -- **Built on @pgpm/object-store**: Inherits content-addressable hashing, structural sharing, and immutability +- **Commits**: Track changes with parent references, messages, and timestamps +- **Refs / Branches**: Named pointers to commits (like Git branches) +- **Stores**: Isolated repositories within a single database +- **Rev-Parse**: Resolve a ref name to the current tree root +- **Set-and-Commit**: Atomic insert-and-commit in a single operation +- **Set-Props-and-Commit**: Update node properties and commit atomically +- **Time Travel**: Access any historical state via commit IDs +- **Built on @pgpm/object-store**: Inherits content-addressable hashing and structural sharing ## Installation @@ -89,265 +85,88 @@ pgpm deploy --createdb --database mydb1 ### Stores -A store is an isolated object repository within a scope, analogous to a Git repository. Multiple stores can exist within the same scope for different purposes (e.g., one for metaschema definitions, another for migration state). Each store has its own independent commit history and refs. +A store is an isolated object repository within a database, analogous to a Git repository. Multiple stores can exist within the same database for different purposes (e.g., metaschema, migrations). ### Commits A commit records a snapshot of the tree at a point in time. Each commit references: - -| Field | Type | Description | -|-------|------|-------------| -| `id` | uuid | Unique commit identifier | -| `scope_id` | uuid | Tenant/namespace isolation | -| `store_id` | uuid | Which store this commit belongs to | -| `tree_id` | uuid | Root object ID of the tree at this commit | -| `parent_ids` | uuid[] | Parent commit(s) for history traversal | -| `message` | text | Descriptive commit message | -| `created_at` | timestamptz | When the commit was created | - -Because the `tree_id` points into the object store (which is immutable and content-addressable), the full state of the tree at any commit is always available — it's never overwritten or garbage-collected. +- `tree_id` — the root object of the tree at this commit +- `parent_ids` — parent commit(s) for history traversal +- `store_id` — which store this commit belongs to +- `message` — descriptive commit message ### Refs -A ref is a named pointer to a commit, exactly like a Git branch. The default ref is `main`. When you make a new commit, the ref is updated to point to the new commit — but the old commit (and its tree) remain accessible by ID. - -| Field | Type | Description | -|-------|------|-------------| -| `id` | uuid | Unique ref identifier | -| `scope_id` | uuid | Tenant/namespace isolation | -| `store_id` | uuid | Which store this ref belongs to | -| `name` | text | The ref name (e.g., `main`) | -| `commit_id` | uuid | The commit this ref currently points to | - -### Scope and Store: Two Levels of Isolation - -- **`scope_id`** isolates tenants — it's on every table and prevents cross-tenant access entirely -- **`store_id`** isolates repositories *within* a tenant — different stores have independent commit histories, refs, and trees, but objects in the underlying store can be shared across stores via structural sharing +A ref is a named pointer to a commit (like a Git branch). The default ref is `main`. ## Core Functions -### object_tree_public.init_empty_repo(s_id, store_id) - -Initialize a new repository with an empty root object, a `main` ref, and a first commit. Raises `REPO_EXISTS` if the store already has commits. +### object_tree_public.init_empty_repo(scope_id, store_id) -**Signature:** -```sql -object_tree_public.init_empty_repo( - s_id uuid, -- scope identifier - store_id uuid -- store identifier for the new repo -) RETURNS void -``` +Initialize an empty repository with a root object, initial commit, and `main` ref. -**Usage:** ```sql SELECT object_tree_public.init_empty_repo( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' + s_id := 'aaaaaaaa-...'::uuid, + store_id := 'bbbbbbbb-...'::uuid ); --- Creates: empty root object, 'main' ref, and first commit ``` -### object_tree_public.set_and_commit(s_id, store_id, refname, path, data, kids, ktree) +### object_tree_public.set_and_commit(scope_id, store_id, refname, path, data, kids, ktree) -Insert or replace a node at the given path and create a new commit in one atomic operation. Resolves the current ref, inserts the node into the current tree (via `object_store_public.insert_node_at_path`), creates a commit with the new tree root, and advances the ref. +Insert a node at the given path and create a new commit atomically. -**Signature:** ```sql -object_tree_public.set_and_commit( - s_id uuid, -- scope identifier - store_id uuid, -- store identifier - refname text, -- ref to commit on (e.g., 'main') - path text[], -- path to the target node - data jsonb, -- JSON payload for the new node - kids uuid[], -- child object IDs - ktree text[] -- child names -) RETURNS uuid -- new tree root ID -``` - -**Usage:** -```sql --- Add a file and commit in one step SELECT object_tree_public.set_and_commit( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', - 'main', - ARRAY['src', 'main.ts'], - '{"content": "hello world"}'::jsonb, - ARRAY[]::uuid[], - ARRAY[]::text[] + s_id := 'aaaaaaaa-...'::uuid, + store_id := 'bbbbbbbb-...'::uuid, + refname := 'main', + path := ARRAY['src', 'main.ts']::text[], + data := '{"content": "hello world"}'::jsonb, + kids := ARRAY[]::uuid[], + ktree := ARRAY[]::text[] ); ``` -### object_tree_public.set_props_and_commit(s_id, store_id, refname, path, data) - -Update the data on an existing node (preserving its children) and commit atomically. Uses `object_store_public.set_data_at_path` under the hood. +### object_tree_public.set_props_and_commit(scope_id, store_id, refname, path, data) -**Signature:** -```sql -object_tree_public.set_props_and_commit( - s_id uuid, -- scope identifier - store_id uuid, -- store identifier - refname text, -- ref to commit on (e.g., 'main') - path text[], -- path to the target node - data jsonb -- new JSON payload (children preserved) -) RETURNS uuid -- new tree root ID -``` +Update the properties of an existing node (preserving children) and commit. -**Usage:** ```sql --- Update properties on an existing node and commit SELECT object_tree_public.set_props_and_commit( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', - 'main', - ARRAY['src', 'main.ts'], - '{"content": "updated content"}'::jsonb + s_id := 'aaaaaaaa-...'::uuid, + store_id := 'bbbbbbbb-...'::uuid, + refname := 'main', + path := ARRAY['src', 'main.ts']::text[], + data := '{"content": "updated"}'::jsonb ); ``` -### object_tree_public.rev_parse(s_id, store_id, refname) +### object_tree_public.rev_parse(scope_id, store_id, refname) -Resolve a ref name to its current tree root ID. Follows the ref to its commit, then returns the commit's `tree_id`. The `refname` parameter defaults to `'main'`. +Resolve a ref name to its current tree root ID. -**Signature:** ```sql -object_tree_public.rev_parse( - s_id uuid, -- scope identifier - store_id uuid, -- store identifier - refname text -- ref name to resolve (default: 'main') -) RETURNS uuid -- tree root ID -``` - -**Usage:** -```sql --- Get the current tree root for 'main' SELECT object_tree_public.rev_parse( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', - 'main' + s_id := 'aaaaaaaa-...'::uuid, + store_id := 'bbbbbbbb-...'::uuid, + refname := 'main' ); ``` -### object_tree_public.get_object_at_path(s_id, store_id, path, refname) +### object_tree_public.get_object_at_path(scope_id, store_id, path, refname) -Get the object at a path for a given ref. Combines `rev_parse` and `object_store_public.get_node_at_path` into a single call. The `refname` parameter defaults to `'main'`. +Get the object at a path for a given ref. -**Signature:** ```sql -object_tree_public.get_object_at_path( - s_id uuid, -- scope identifier - store_id uuid, -- store identifier - path text[], -- path to the target node - refname text -- ref name (default: 'main') -) RETURNS object_store_public.object -``` - -**Usage:** -```sql --- Get the object at a path on the main branch SELECT * FROM object_tree_public.get_object_at_path( - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', - ARRAY['src', 'main.ts'], - 'main' -); -``` - -## Usage Examples - -### Setting Up a Repository and Making Changes - -```sql -DO $$ -DECLARE - scope uuid := 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'; - store uuid := 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'; - tree_root uuid; -BEGIN - -- Initialize a new repo - PERFORM object_tree_public.init_empty_repo(scope, store); - - -- Add a file and commit - SELECT object_tree_public.set_and_commit( - scope, store, 'main', - ARRAY['README.md'], - '{"content": "# My Project"}'::jsonb, - ARRAY[]::uuid[], ARRAY[]::text[] - ) INTO tree_root; - - -- Add another file and commit - SELECT object_tree_public.set_and_commit( - scope, store, 'main', - ARRAY['src', 'index.ts'], - '{"content": "export default {}"}'::jsonb, - ARRAY[]::uuid[], ARRAY[]::text[] - ) INTO tree_root; - - -- Update a file's content (preserving children) and commit - SELECT object_tree_public.set_props_and_commit( - scope, store, 'main', - ARRAY['README.md'], - '{"content": "# My Project\n\nUpdated readme."}'::jsonb - ) INTO tree_root; -END $$; -``` - -### Time Travel: Reading Historical State - -```sql --- Every commit stores its tree_id, so you can read any past state: - --- Get all commits for a store (newest first) -SELECT id, tree_id, message, created_at -FROM object_tree_public.commit -WHERE scope_id = 'aaaaaaaa-...' AND store_id = 'bbbbbbbb-...' -ORDER BY created_at DESC; - --- Read the tree at any historical commit -SELECT * FROM object_store_public.get_all( - 'aaaaaaaa-...', - 'historical-tree-id-from-commit'::uuid + s_id := 'aaaaaaaa-...'::uuid, + store_id := 'bbbbbbbb-...'::uuid, + path := ARRAY['src', 'main.ts']::text[], + refname := 'main' ); ``` -### Multiple Stores in One Scope - -```sql --- Store 1: metaschema definitions -SELECT object_tree_public.init_empty_repo(scope, store_metaschema); -SELECT object_tree_public.set_and_commit( - scope, store_metaschema, 'main', - ARRAY['tables', 'users'], - '{"columns": ["id", "email", "name"]}'::jsonb, - ARRAY[]::uuid[], ARRAY[]::text[] -); - --- Store 2: migration state (completely independent history) -SELECT object_tree_public.init_empty_repo(scope, store_migrations); -SELECT object_tree_public.set_and_commit( - scope, store_migrations, 'main', - ARRAY['001_init'], - '{"applied": true, "at": "2026-01-15"}'::jsonb, - ARRAY[]::uuid[], ARRAY[]::text[] -); -``` - -## Database Schema - -### Tables - -| Table | Schema | Purpose | -|-------|--------|---------| -| `commit` | `object_tree_public` | Commit history with tree snapshots and parent links | -| `ref` | `object_tree_public` | Named pointers (branches) to commits | -| `store` | `object_tree_public` | Store registry (optional metadata for stores) | - -### Schemas - -| Schema | Purpose | -|--------|---------| -| `object_tree_public` | Public API — commits, refs, stores, and all user-facing functions | - ## Testing ```bash @@ -356,18 +175,13 @@ pnpm test ## Dependencies -- [`@pgpm/object-store`](https://www.npmjs.com/package/@pgpm/object-store): Content-addressable Merkle tree storage — the immutable foundation this package builds on -- [`@pgpm/verify`](https://www.npmjs.com/package/@pgpm/verify): Verification utilities +- `@pgpm/object-store`: Content-addressable Merkle tree storage +- `@pgpm/verify`: Verification utilities ## Related Tooling -* [pgpm](https://github.com/constructive-io/constructive/tree/main/packages/pgpm): **PostgreSQL Package Manager** for modular Postgres development. Works with database workspaces, scaffolding, migrations, seeding, and installing database packages. -* [pgsql-test](https://github.com/constructive-io/constructive/tree/main/packages/pgsql-test): **Isolated testing environments** with per-test transaction rollbacks — ideal for integration tests, complex migrations, and RLS simulation. -* [supabase-test](https://github.com/constructive-io/constructive/tree/main/packages/supabase-test): **Supabase-native test harness** preconfigured for the local Supabase stack — per-test rollbacks, JWT/role context helpers, and CI/GitHub Actions ready. -* [graphile-test](https://github.com/constructive-io/constructive/tree/main/packages/graphile-test): **Authentication mocking** for Graphile-focused test helpers and emulating row-level security contexts. -* [pgsql-parser](https://github.com/constructive-io/pgsql-parser): **SQL conversion engine** that interprets and converts PostgreSQL syntax. -* [libpg-query-node](https://github.com/constructive-io/libpg-query-node): **Node.js bindings** for `libpg_query`, converting SQL into parse trees. -* [pg-proto-parser](https://github.com/constructive-io/pg-proto-parser): **Protobuf parser** for parsing PostgreSQL Protocol Buffers definitions to generate TypeScript interfaces, utility functions, and JSON mappings for enums. +* [pgpm](https://github.com/constructive-io/constructive/tree/main/packages/pgpm): PostgreSQL Package Manager for modular Postgres development. +* [pgsql-test](https://github.com/constructive-io/constructive/tree/main/packages/pgsql-test): Isolated testing environments with per-test transaction rollbacks. ## Disclaimer diff --git a/packages/object-tree/object-tree.control b/packages/object-tree/object-tree.control index 400b1052..3cf576ba 100644 --- a/packages/object-tree/object-tree.control +++ b/packages/object-tree/object-tree.control @@ -1,6 +1,6 @@ # object-tree extension comment = 'object-tree extension - git-like version control for database objects' -default_version = '0.15.5' +default_version = '0.26.0' module_pathname = '$libdir/object-tree' requires = 'plpgsql,pgcrypto,object-store,pgpm-verify' relocatable = false diff --git a/packages/object-tree/package.json b/packages/object-tree/package.json index fdbe8518..4f4a4beb 100644 --- a/packages/object-tree/package.json +++ b/packages/object-tree/package.json @@ -38,4 +38,4 @@ "bugs": { "url": "https://github.com/constructive-io/pgpm-modules/issues" } -} +} \ No newline at end of file diff --git a/packages/object-tree/sql/object-tree--0.15.5.sql b/packages/object-tree/sql/object-tree--0.26.0.sql similarity index 93% rename from packages/object-tree/sql/object-tree--0.15.5.sql rename to packages/object-tree/sql/object-tree--0.26.0.sql index f6c85741..0b27cdac 100644 --- a/packages/object-tree/sql/object-tree--0.15.5.sql +++ b/packages/object-tree/sql/object-tree--0.26.0.sql @@ -116,8 +116,8 @@ DECLARE tree_id uuid; obj object_store_public.object; BEGIN - tree_id = object_tree_public.rev_parse(s_id, store_id, refname); - SELECT * FROM object_store_public.get_node_at_path(s_id, tree_id, path) + tree_id = object_tree_public.rev_parse(s_id, store_id, refname); + SELECT * FROM object_store_public.get_node_at_path(s_id, tree_id, path) INTO obj; RETURN obj; END; @@ -138,16 +138,16 @@ BEGIN RAISE EXCEPTION 'REPO_EXISTS'; END IF; - INSERT INTO object_store_public.object (scope_id) + INSERT INTO object_store_public.object (scope_id) VALUES (s_id) - RETURNING id INTO vtree_id; + RETURNING id INTO vtree_id; - INSERT INTO object_tree_public.ref (scope_id, store_id, name) - VALUES (s_id, init_empty_repo.store_id, 'main') + INSERT INTO object_tree_public.ref (scope_id, store_id, name) + VALUES (s_id, init_empty_repo.store_id, 'main') RETURNING id INTO vref_id; - INSERT INTO object_tree_public.commit (scope_id, store_id, message, tree_id) - VALUES (s_id, init_empty_repo.store_id, 'first commit', vtree_id) + INSERT INTO object_tree_public.commit (scope_id, store_id, message, tree_id) + VALUES (s_id, init_empty_repo.store_id, 'first commit', vtree_id) RETURNING id into vcommit_id; UPDATE object_tree_public.ref SET commit_id = vcommit_id @@ -304,4 +304,4 @@ COMMENT ON COLUMN object_tree_public.store.scope_id IS 'The scope this store bel COMMENT ON COLUMN object_tree_public.store.hash IS 'The current head tree_id for this store.'; -CREATE UNIQUE INDEX idx_unique_store_name ON object_tree_public.store (scope_id, (decode(md5(lower(name)), 'hex'))); +CREATE UNIQUE INDEX idx_unique_store_name ON object_tree_public.store (scope_id, (decode(md5(lower(name)), 'hex'))); \ No newline at end of file diff --git a/packages/partman/Makefile b/packages/partman/Makefile index b65cde02..12beba90 100644 --- a/packages/partman/Makefile +++ b/packages/partman/Makefile @@ -1,5 +1,5 @@ EXTENSION = pgpm-partman -DATA = sql/pgpm-partman--0.0.1.sql +DATA = sql/pgpm-partman--0.26.0.sql PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/packages/partman/deploy/extensions/pg_partman.sql b/packages/partman/deploy/extensions/pg_partman.sql index a9c6fc75..10a63c13 100644 --- a/packages/partman/deploy/extensions/pg_partman.sql +++ b/packages/partman/deploy/extensions/pg_partman.sql @@ -6,3 +6,7 @@ BEGIN EXECUTE 'CREATE EXTENSION pg_partman SCHEMA partman'; END; $$; + +GRANT USAGE ON SCHEMA partman TO authenticated; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO authenticated; +ALTER DEFAULT PRIVILEGES IN SCHEMA partman GRANT EXECUTE ON FUNCTIONS TO authenticated; diff --git a/packages/partman/package.json b/packages/partman/package.json index ad4c5788..890c2e01 100644 --- a/packages/partman/package.json +++ b/packages/partman/package.json @@ -32,4 +32,4 @@ "bugs": { "url": "https://github.com/constructive-io/pgpm-modules/issues" } -} +} \ No newline at end of file diff --git a/packages/partman/pgpm-partman.control b/packages/partman/pgpm-partman.control index 39a46156..cd0b2fa1 100644 --- a/packages/partman/pgpm-partman.control +++ b/packages/partman/pgpm-partman.control @@ -1,6 +1,6 @@ # pgpm-partman extension comment = 'pg_partman wrapper - installs pg_partman into partman schema' -default_version = '0.0.1' +default_version = '0.26.0' module_pathname = '$libdir/pgpm-partman' requires = 'plpgsql,metaschema-schema' relocatable = false diff --git a/packages/partman/sql/pgpm-partman--0.0.1.sql b/packages/partman/sql/pgpm-partman--0.26.0.sql similarity index 72% rename from packages/partman/sql/pgpm-partman--0.0.1.sql rename to packages/partman/sql/pgpm-partman--0.26.0.sql index be912cb2..ecbcc651 100644 --- a/packages/partman/sql/pgpm-partman--0.0.1.sql +++ b/packages/partman/sql/pgpm-partman--0.26.0.sql @@ -6,7 +6,22 @@ BEGIN END; $EOFCODE$; -CREATE FUNCTION partman.create_parent_with_retention(v_parent_table text, v_control text, v_type text DEFAULT 'range', partition_interval text DEFAULT '1 day', v_premake int DEFAULT 2, v_retention text DEFAULT NULL, v_retention_keep_table boolean DEFAULT true) RETURNS void AS $EOFCODE$ +GRANT USAGE ON SCHEMA partman TO authenticated; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO authenticated; + +ALTER DEFAULT PRIVILEGES IN SCHEMA partman + GRANT EXECUTE ON FUNCTIONS TO authenticated; + +CREATE FUNCTION partman.create_parent_with_retention( + v_parent_table text, + v_control text, + v_type text DEFAULT 'range', + partition_interval text DEFAULT '1 day', + v_premake int DEFAULT 2, + v_retention text DEFAULT NULL, + v_retention_keep_table boolean DEFAULT true +) RETURNS void AS $EOFCODE$ BEGIN PERFORM partman.create_parent( p_parent_table := v_parent_table, @@ -25,7 +40,15 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE; -CREATE FUNCTION partman.create_parent_by_id(v_table_id uuid, v_control text, v_type text DEFAULT 'range', partition_interval text DEFAULT '1 day', v_premake int DEFAULT 2, v_retention text DEFAULT NULL, v_retention_keep_table boolean DEFAULT true) RETURNS void AS $EOFCODE$ +CREATE FUNCTION partman.create_parent_by_id( + v_table_id uuid, + v_control text, + v_type text DEFAULT 'range', + partition_interval text DEFAULT '1 day', + v_premake int DEFAULT 2, + v_retention text DEFAULT NULL, + v_retention_keep_table boolean DEFAULT true +) RETURNS void AS $EOFCODE$ DECLARE v_parent_table text; BEGIN @@ -51,7 +74,9 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE; -CREATE FUNCTION partman.remove_parent_by_id(v_table_id uuid) RETURNS void AS $EOFCODE$ +CREATE FUNCTION partman.remove_parent_by_id( + v_table_id uuid +) RETURNS void AS $EOFCODE$ DECLARE v_parent_table text; BEGIN @@ -70,7 +95,9 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql VOLATILE; -CREATE FUNCTION partman.verify_parent_by_id(v_table_id uuid) RETURNS boolean AS $EOFCODE$ +CREATE FUNCTION partman.verify_parent_by_id( + v_table_id uuid +) RETURNS boolean AS $EOFCODE$ DECLARE v_parent_table text; v_found boolean; @@ -94,7 +121,10 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql STABLE; -CREATE FUNCTION partman.run_maintenance_by_id(v_table_id uuid DEFAULT NULL, v_analyze boolean DEFAULT true) RETURNS void AS $EOFCODE$ +CREATE FUNCTION partman.run_maintenance_by_id( + v_table_id uuid DEFAULT NULL, + v_analyze boolean DEFAULT true +) RETURNS void AS $EOFCODE$ DECLARE v_parent_table text; BEGIN