From ebd44ba58226ba78219c3d9dbf56fd8673acdfa1 Mon Sep 17 00:00:00 2001 From: Wojtek Majewski Date: Wed, 7 Jan 2026 12:04:37 +0100 Subject: [PATCH] introduce pgflow auth secret key --- .changeset/add-pgflow-auth-secret-support.md | 6 + .../schemas/0059_function_ensure_workers.sql | 8 +- ...60124113408_pgflow_auth_secret_support.sql | 107 ++++++++++++++++++ pkgs/core/supabase/migrations/atlas.sum | 3 +- .../credentials_pgflow_auth_secret.test.sql | 80 +++++++++++++ pkgs/edge-worker/src/shared/authValidation.ts | 11 +- .../tests/unit/shared/authValidation.test.ts | 72 +++++++++++- .../deploy/supabase/configure-secrets.mdx | 66 ++++++++++- ...fix-for-ensure-workers-and-edge-worker.mdx | 64 +++++++++++ 9 files changed, 407 insertions(+), 10 deletions(-) create mode 100644 .changeset/add-pgflow-auth-secret-support.md create mode 100644 pkgs/core/supabase/migrations/20260124113408_pgflow_auth_secret_support.sql create mode 100644 pkgs/core/supabase/tests/ensure_workers/credentials_pgflow_auth_secret.test.sql create mode 100644 pkgs/website/src/content/docs/news/pgflow-0-13-3-authentication-fix-for-ensure-workers-and-edge-worker.mdx diff --git a/.changeset/add-pgflow-auth-secret-support.md b/.changeset/add-pgflow-auth-secret-support.md new file mode 100644 index 000000000..ff7224bd1 --- /dev/null +++ b/.changeset/add-pgflow-auth-secret-support.md @@ -0,0 +1,6 @@ +--- +'@pgflow/core': patch +'@pgflow/edge-worker': patch +--- + +Add PGFLOW_AUTH_SECRET support to bypass JWT format mismatch in ensure_workers authentication diff --git a/pkgs/core/schemas/0059_function_ensure_workers.sql b/pkgs/core/schemas/0059_function_ensure_workers.sql index f32dbeb03..3dc3d3987 100644 --- a/pkgs/core/schemas/0059_function_ensure_workers.sql +++ b/pkgs/core/schemas/0059_function_ensure_workers.sql @@ -14,11 +14,15 @@ as $$ -- Get credentials: Local mode uses hardcoded URL, production uses vault secrets -- Empty strings are treated as NULL using nullif() + -- pgflow_auth_secret takes priority over supabase_service_role_key for production auth credentials as ( select case when (select is_local from env) then null - else nullif((select decrypted_secret from vault.decrypted_secrets where name = 'supabase_service_role_key'), '') + else coalesce( + nullif((select decrypted_secret from vault.decrypted_secrets where name = 'pgflow_auth_secret'), ''), + nullif((select decrypted_secret from vault.decrypted_secrets where name = 'supabase_service_role_key'), '') + ) end as service_role_key, case when (select is_local from env) then 'http://kong:8000/functions/v1' @@ -105,6 +109,6 @@ comment on function pgflow.ensure_workers() is In local mode: pings ALL enabled functions (ignores debounce AND alive workers check). In production mode: only pings functions that pass debounce AND have no alive workers. Debounce: skips functions pinged within their debounce interval (production only). -Credentials: Uses Vault secrets (supabase_service_role_key, supabase_project_id) or local fallbacks. +Credentials: Uses Vault secrets (pgflow_auth_secret with fallback to supabase_service_role_key, supabase_project_id) or local fallbacks. URL is built from project_id: https://{project_id}.supabase.co/functions/v1 Returns request_id from pg_net for each HTTP request made.'; diff --git a/pkgs/core/supabase/migrations/20260124113408_pgflow_auth_secret_support.sql b/pkgs/core/supabase/migrations/20260124113408_pgflow_auth_secret_support.sql new file mode 100644 index 000000000..7114572df --- /dev/null +++ b/pkgs/core/supabase/migrations/20260124113408_pgflow_auth_secret_support.sql @@ -0,0 +1,107 @@ +-- Modify "ensure_workers" function +CREATE OR REPLACE FUNCTION "pgflow"."ensure_workers" () RETURNS TABLE ("function_name" text, "invoked" boolean, "request_id" bigint) LANGUAGE sql AS $$ +with + -- Detect environment + env as ( + select pgflow.is_local() as is_local + ), + + -- Get credentials: Local mode uses hardcoded URL, production uses vault secrets + -- Empty strings are treated as NULL using nullif() + -- pgflow_auth_secret takes priority over supabase_service_role_key for production auth + credentials as ( + select + case + when (select is_local from env) then null + else coalesce( + nullif((select decrypted_secret from vault.decrypted_secrets where name = 'pgflow_auth_secret'), ''), + nullif((select decrypted_secret from vault.decrypted_secrets where name = 'supabase_service_role_key'), '') + ) + end as service_role_key, + case + when (select is_local from env) then 'http://kong:8000/functions/v1' + else (select 'https://' || nullif(decrypted_secret, '') || '.supabase.co/functions/v1' from vault.decrypted_secrets where name = 'supabase_project_id') + end as base_url + ), + + -- Find functions that pass the debounce check + debounce_passed as ( + select wf.function_name, wf.debounce + from pgflow.worker_functions as wf + where wf.enabled = true + and ( + wf.last_invoked_at is null + or wf.last_invoked_at < now() - wf.debounce + ) + ), + + -- Find functions that have at least one alive worker + functions_with_alive_workers as ( + select distinct w.function_name + from pgflow.workers as w + inner join debounce_passed as dp on w.function_name = dp.function_name + where w.stopped_at is null + and w.deprecated_at is null + and w.last_heartbeat_at > now() - dp.debounce + ), + + -- Determine which functions should be invoked + -- Local mode: all enabled functions (bypass debounce AND alive workers check) + -- Production mode: only functions that pass debounce AND have no alive workers + functions_to_invoke as ( + select wf.function_name + from pgflow.worker_functions as wf + where wf.enabled = true + and ( + pgflow.is_local() = true -- Local: all enabled functions + or ( + -- Production: debounce + no alive workers + wf.function_name in (select dp.function_name from debounce_passed as dp) + and wf.function_name not in (select faw.function_name from functions_with_alive_workers as faw) + ) + ) + ), + + -- Make HTTP requests and capture request_ids + http_requests as ( + select + fti.function_name, + net.http_post( + url => c.base_url || '/' || fti.function_name, + headers => case + when e.is_local then '{}'::jsonb + else jsonb_build_object( + 'Content-Type', 'application/json', + 'Authorization', 'Bearer ' || c.service_role_key + ) + end, + body => '{}'::jsonb + ) as request_id + from functions_to_invoke as fti + cross join credentials as c + cross join env as e + where c.base_url is not null + and (e.is_local or c.service_role_key is not null) + ), + + -- Update last_invoked_at for invoked functions + updated as ( + update pgflow.worker_functions as wf + set last_invoked_at = clock_timestamp() + from http_requests as hr + where wf.function_name = hr.function_name + returning wf.function_name + ) + + select u.function_name, true as invoked, hr.request_id + from updated as u + inner join http_requests as hr on u.function_name = hr.function_name +$$; +-- Set comment to function: "ensure_workers" +COMMENT ON FUNCTION "pgflow"."ensure_workers" IS 'Ensures worker functions are running by pinging them via HTTP when needed. +In local mode: pings ALL enabled functions (ignores debounce AND alive workers check). +In production mode: only pings functions that pass debounce AND have no alive workers. +Debounce: skips functions pinged within their debounce interval (production only). +Credentials: Uses Vault secrets (pgflow_auth_secret with fallback to supabase_service_role_key, supabase_project_id) or local fallbacks. +URL is built from project_id: https://{project_id}.supabase.co/functions/v1 +Returns request_id from pg_net for each HTTP request made.'; diff --git a/pkgs/core/supabase/migrations/atlas.sum b/pkgs/core/supabase/migrations/atlas.sum index 1d66f78c4..25b6ab286 100644 --- a/pkgs/core/supabase/migrations/atlas.sum +++ b/pkgs/core/supabase/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:dzKOHL+hbunxWTZaGOIDWQG9THDva7Pk7VVDGASJkps= +h1:q21PG0IM91BaFRI7KM/qP5t76ER7If38sCU9TieHenI= 20250429164909_pgflow_initial.sql h1:I3n/tQIg5Q5nLg7RDoU3BzqHvFVjmumQxVNbXTPG15s= 20250517072017_pgflow_fix_poll_for_tasks_to_use_separate_statement_for_polling.sql h1:wTuXuwMxVniCr3ONCpodpVWJcHktoQZIbqMZ3sUHKMY= 20250609105135_pgflow_add_start_tasks_and_started_status.sql h1:ggGanW4Wyt8Kv6TWjnZ00/qVb3sm+/eFVDjGfT8qyPg= @@ -17,3 +17,4 @@ h1:dzKOHL+hbunxWTZaGOIDWQG9THDva7Pk7VVDGASJkps= 20251225163110_pgflow_add_flow_input_column.sql h1:734uCbTgKmPhTK3TY56uNYZ31T8u59yll9ea7nwtEoc= 20260103145141_pgflow_step_output_storage.sql h1:mgVHSFDLdtYy//SZ6C03j9Str1iS9xCM8Rz/wyFwn3o= 20260120205547_pgflow_requeue_stalled_tasks.sql h1:4wCBBvjtETCgJf1eXmlH5wCTKDUhiLi0uzsFG1V528E= +20260124113408_pgflow_auth_secret_support.sql h1:i/s1JkBqRElN6FOYFQviJt685W08SuSo30aP25lNlLc= diff --git a/pkgs/core/supabase/tests/ensure_workers/credentials_pgflow_auth_secret.test.sql b/pkgs/core/supabase/tests/ensure_workers/credentials_pgflow_auth_secret.test.sql new file mode 100644 index 000000000..596bf5e5a --- /dev/null +++ b/pkgs/core/supabase/tests/ensure_workers/credentials_pgflow_auth_secret.test.sql @@ -0,0 +1,80 @@ +-- Test: ensure_workers() uses pgflow_auth_secret with fallback to supabase_service_role_key +begin; +select plan(3); +select pgflow_tests.reset_db(); + +-- Setup: Create project ID secret (needed for all tests) +select vault.create_secret('testproject123', 'supabase_project_id'); + +-- Setup: Register a worker function +select pgflow.track_worker_function('my-function'); + +-- Simulate production mode +set local app.settings.jwt_secret = 'production-secret-different-from-local'; + +-- ============================================================================= +-- TEST 1: pgflow_auth_secret takes priority when both are set +-- ============================================================================= +select vault.create_secret('pgflow-auth-secret-value', 'pgflow_auth_secret'); +select vault.create_secret('legacy-service-role-key', 'supabase_service_role_key'); + +update pgflow.worker_functions +set last_invoked_at = now() - interval '10 seconds'; + +select * into temporary test1_result from pgflow.ensure_workers(); + +select ok( + (select headers->>'Authorization' = 'Bearer pgflow-auth-secret-value' + from net.http_request_queue + where id = (select request_id from test1_result limit 1)), + 'pgflow_auth_secret takes priority over supabase_service_role_key' +); + +drop table test1_result; + +-- Cleanup secrets for next test +delete from vault.secrets where name in ('pgflow_auth_secret', 'supabase_service_role_key'); + +-- ============================================================================= +-- TEST 2: Falls back to supabase_service_role_key when pgflow_auth_secret not set +-- ============================================================================= +select vault.create_secret('fallback-service-role-key', 'supabase_service_role_key'); + +update pgflow.worker_functions +set last_invoked_at = now() - interval '10 seconds'; + +select * into temporary test2_result from pgflow.ensure_workers(); + +select ok( + (select headers->>'Authorization' = 'Bearer fallback-service-role-key' + from net.http_request_queue + where id = (select request_id from test2_result limit 1)), + 'Falls back to supabase_service_role_key when pgflow_auth_secret not set' +); + +drop table test2_result; + +-- Cleanup secrets for next test +delete from vault.secrets where name = 'supabase_service_role_key'; + +-- ============================================================================= +-- TEST 3: pgflow_auth_secret works without supabase_service_role_key +-- ============================================================================= +select vault.create_secret('standalone-auth-secret', 'pgflow_auth_secret'); + +update pgflow.worker_functions +set last_invoked_at = now() - interval '10 seconds'; + +select * into temporary test3_result from pgflow.ensure_workers(); + +select ok( + (select headers->>'Authorization' = 'Bearer standalone-auth-secret' + from net.http_request_queue + where id = (select request_id from test3_result limit 1)), + 'pgflow_auth_secret works without supabase_service_role_key being set' +); + +drop table test3_result; + +select finish(); +rollback; diff --git a/pkgs/edge-worker/src/shared/authValidation.ts b/pkgs/edge-worker/src/shared/authValidation.ts index 75b92ec2e..493a34758 100644 --- a/pkgs/edge-worker/src/shared/authValidation.ts +++ b/pkgs/edge-worker/src/shared/authValidation.ts @@ -16,14 +16,19 @@ export function validateServiceRoleAuth( } const authHeader = request.headers.get('Authorization'); - const expectedKey = env['SUPABASE_SERVICE_ROLE_KEY']; + + // Treat empty string as unset - use PGFLOW_AUTH_SECRET if set and non-empty, + // otherwise fall back to SUPABASE_SERVICE_ROLE_KEY + const authSecret = env['PGFLOW_AUTH_SECRET']; + const serviceRoleKey = env['SUPABASE_SERVICE_ROLE_KEY']; + const expectedKey = (authSecret && authSecret !== '') ? authSecret : serviceRoleKey; if (!authHeader) { return { valid: false, error: 'Missing Authorization header' }; } - if (!expectedKey) { - return { valid: false, error: 'Server misconfigured: missing service role key' }; + if (!expectedKey || expectedKey === '') { + return { valid: false, error: 'Server misconfigured: missing PGFLOW_AUTH_SECRET or SUPABASE_SERVICE_ROLE_KEY' }; } const expected = `Bearer ${expectedKey}`; diff --git a/pkgs/edge-worker/tests/unit/shared/authValidation.test.ts b/pkgs/edge-worker/tests/unit/shared/authValidation.test.ts index cf7398871..dbfd74f03 100644 --- a/pkgs/edge-worker/tests/unit/shared/authValidation.test.ts +++ b/pkgs/edge-worker/tests/unit/shared/authValidation.test.ts @@ -31,7 +31,19 @@ function productionEnv(serviceRoleKey?: string): Record { + return { + SUPABASE_ANON_KEY: 'production-anon-key-abc', + SUPABASE_SERVICE_ROLE_KEY: serviceRoleKey, + PGFLOW_AUTH_SECRET: authSecret, + }; +} + const PRODUCTION_SERVICE_ROLE_KEY = 'production-service-role-key-xyz'; +const PGFLOW_AUTH_SECRET_VALUE = 'user-controlled-auth-secret-123'; // ============================================================ // validateServiceRoleAuth() - Local mode tests @@ -80,7 +92,7 @@ Deno.test('validateServiceRoleAuth - production: accepts request with correct au Deno.test('validateServiceRoleAuth - production: rejects when service role key not configured', () => { const request = createRequest('Bearer any-key'); const result = validateServiceRoleAuth(request, productionEnv(undefined)); - assertEquals(result, { valid: false, error: 'Server misconfigured: missing service role key' }); + assertEquals(result, { valid: false, error: 'Server misconfigured: missing PGFLOW_AUTH_SECRET or SUPABASE_SERVICE_ROLE_KEY' }); }); Deno.test('validateServiceRoleAuth - production: rejects Basic auth scheme', () => { @@ -101,6 +113,64 @@ Deno.test('validateServiceRoleAuth - production: rejects auth header without sch assertEquals(result, { valid: false, error: 'Invalid Authorization header' }); }); +// ============================================================ +// validateServiceRoleAuth() - PGFLOW_AUTH_SECRET tests +// ============================================================ + +Deno.test('validateServiceRoleAuth - PGFLOW_AUTH_SECRET: accepts request with auth secret when set', () => { + const request = createRequest(`Bearer ${PGFLOW_AUTH_SECRET_VALUE}`); + const result = validateServiceRoleAuth( + request, + productionEnvWithAuthSecret(PGFLOW_AUTH_SECRET_VALUE, PRODUCTION_SERVICE_ROLE_KEY) + ); + assertEquals(result, { valid: true }); +}); + +Deno.test('validateServiceRoleAuth - PGFLOW_AUTH_SECRET: rejects service role key when auth secret is set', () => { + const request = createRequest(`Bearer ${PRODUCTION_SERVICE_ROLE_KEY}`); + const result = validateServiceRoleAuth( + request, + productionEnvWithAuthSecret(PGFLOW_AUTH_SECRET_VALUE, PRODUCTION_SERVICE_ROLE_KEY) + ); + assertEquals(result, { valid: false, error: 'Invalid Authorization header' }); +}); + +Deno.test('validateServiceRoleAuth - PGFLOW_AUTH_SECRET: falls back to service role key when auth secret not set', () => { + const request = createRequest(`Bearer ${PRODUCTION_SERVICE_ROLE_KEY}`); + const result = validateServiceRoleAuth( + request, + productionEnvWithAuthSecret(undefined, PRODUCTION_SERVICE_ROLE_KEY) + ); + assertEquals(result, { valid: true }); +}); + +Deno.test('validateServiceRoleAuth - PGFLOW_AUTH_SECRET: works without service role key when auth secret is set', () => { + const request = createRequest(`Bearer ${PGFLOW_AUTH_SECRET_VALUE}`); + const result = validateServiceRoleAuth( + request, + productionEnvWithAuthSecret(PGFLOW_AUTH_SECRET_VALUE, undefined) + ); + assertEquals(result, { valid: true }); +}); + +Deno.test('validateServiceRoleAuth - PGFLOW_AUTH_SECRET: returns error when neither key is set', () => { + const request = createRequest('Bearer any-key'); + const result = validateServiceRoleAuth( + request, + productionEnvWithAuthSecret(undefined, undefined) + ); + assertEquals(result, { valid: false, error: 'Server misconfigured: missing PGFLOW_AUTH_SECRET or SUPABASE_SERVICE_ROLE_KEY' }); +}); + +Deno.test('validateServiceRoleAuth - PGFLOW_AUTH_SECRET: treats empty string as unset, falls back to service role key', () => { + const request = createRequest(`Bearer ${PRODUCTION_SERVICE_ROLE_KEY}`); + const result = validateServiceRoleAuth( + request, + productionEnvWithAuthSecret('', PRODUCTION_SERVICE_ROLE_KEY) // Empty string + ); + assertEquals(result, { valid: true }); +}); + // ============================================================ // createUnauthorizedResponse() tests // ============================================================ diff --git a/pkgs/website/src/content/docs/deploy/supabase/configure-secrets.mdx b/pkgs/website/src/content/docs/deploy/supabase/configure-secrets.mdx index 58843c18f..f7b749a51 100644 --- a/pkgs/website/src/content/docs/deploy/supabase/configure-secrets.mdx +++ b/pkgs/website/src/content/docs/deploy/supabase/configure-secrets.mdx @@ -5,7 +5,7 @@ sidebar: order: 15 --- -import { Aside, Steps } from "@astrojs/starlight/components"; +import { Aside, Steps, Tabs, TabItem } from '@astrojs/starlight/components'; pgflow needs vault secrets to automatically manage workers in production. @@ -29,18 +29,78 @@ pgflow needs vault secrets to automatically manage workers in production. ![API Keys page showing the service_role key](../../../../assets/deploy/supabase/05-service-role-key.png) 3. ### Create vault secrets + The service role key needs to be stored in two places: + + - **Vault** - so the database can authenticate when calling the worker + - **Edge Function secret** (step 4) - so the worker can validate incoming requests + Run in the SQL Editor: ```sql "your-project-id" "your-service-role-key" SELECT vault.create_secret('your-project-id', 'supabase_project_id'); - SELECT vault.create_secret('your-service-role-key', 'supabase_service_role_key'); + SELECT vault.create_secret('your-service-role-key', 'pgflow_auth_secret'); ``` Replace the highlighted values with your actual Project ID and service role key from steps 1 and 2. +4. ### Set the Edge Function auth secret + + Set `PGFLOW_AUTH_SECRET` to the same service role key you stored in vault: + + + + 1. Go to **Edge Functions** in the sidebar 2. Select **pgflow-worker** + function 3. Click **Manage secrets** 4. Add `PGFLOW_AUTH_SECRET` with + your service role key value + + + ```bash frame="none" supabase secrets set + PGFLOW_AUTH_SECRET=your-service-role-key --project-ref your-project-id + ``` + + + + +## Troubleshooting + +### 401 Unauthorized from worker + +```text del={1} +Invalid Authorization header +``` + +The vault secret and Edge Function secret don't match. Verify both sides: + +**Check vault secret:** + +```sql +SELECT name, decrypted_secret +FROM vault.decrypted_secrets +WHERE name IN ('pgflow_auth_secret', 'supabase_service_role_key'); +``` + +**Check Edge Function secret:** + + + + Go to **Edge Functions** -> **pgflow-worker** -> **Manage secrets** and + verify `PGFLOW_AUTH_SECRET` matches the vault value. + + + ```bash frame="none" supabase secrets list --project-ref your-project-id ``` + + + +### Workers not starting + +If `ensure_workers()` runs but workers don't start, check: + +1. **Vault secrets exist** - Run the SQL query above +2. **Project ID is correct** - The URL is built as `https://{project_id}.supabase.co/functions/v1` +3. **Function is deployed** - Verify with `supabase functions list` diff --git a/pkgs/website/src/content/docs/news/pgflow-0-13-3-authentication-fix-for-ensure-workers-and-edge-worker.mdx b/pkgs/website/src/content/docs/news/pgflow-0-13-3-authentication-fix-for-ensure-workers-and-edge-worker.mdx new file mode 100644 index 000000000..c099b2484 --- /dev/null +++ b/pkgs/website/src/content/docs/news/pgflow-0-13-3-authentication-fix-for-ensure-workers-and-edge-worker.mdx @@ -0,0 +1,64 @@ +--- +draft: false +title: 'pgflow 0.13.3: Authentication Fix for ensure_workers and EdgeWorker' +description: 'Fixes 401 authentication errors between the cron-based worker management and Edge Functions by introducing PGFLOW_AUTH_SECRET' +date: 2026-02-03 +authors: + - jumski +tags: + - bugfix + - authentication + - edge-worker + - release +featured: true +--- + +import { Aside } from '@astrojs/starlight/components'; + +This release fixes a critical authentication issue where `ensure_workers()` fails to start Edge Workers due to key format mismatches, resulting in 401 Unauthorized errors. This resolves [GitHub issue #603](https://github.com/pgflow-dev/pgflow/issues/603). + +## The Problem + +Supabase stores the service role key in different formats depending on where you access it: + +- **In Vault** (used by `ensure_workers`): JWT format starting with `eyJhbG...` +- **In Edge Functions** (`SUPABASE_SERVICE_ROLE_KEY` env var): Internal format starting with `sb_secret_...` + +When `ensure_workers` sends an HTTP request to trigger a worker, it includes the JWT-format key from vault in the Authorization header. However, the Edge Worker validates this against `SUPABASE_SERVICE_ROLE_KEY`, which contains the `sb_secret_...` format. Since these strings don't match, authentication fails with a 401 error. + +## The Solution + +We've introduced `PGFLOW_AUTH_SECRET` - a user-controlled authentication secret that bypasses the format mismatch entirely: + +1. Store your custom secret in vault as `pgflow_auth_secret` +2. Set the same value as `PGFLOW_AUTH_SECRET` in your Edge Function secrets +3. Both sides now use the identical string for authentication + + + +## How to Update + +Update your pgflow installation and configure the new authentication secret: + +1. Update your packages and migrations following the [update guide](/deploy/update-pgflow/) +2. Set `pgflow_auth_secret` in vault (same value you'll use in step 3): + ```sql + SELECT vault.create_secret('your-secret-value', 'pgflow_auth_secret'); + ``` +3. Set `PGFLOW_AUTH_SECRET` in your Edge Function secrets with the same value + +See the [Configure Secrets](/deploy/supabase/configure-secrets/) documentation for detailed setup instructions. + +## Backward Compatibility + +For backwards compatibility, pgflow falls back to the legacy secrets if `PGFLOW_AUTH_SECRET` is not configured: + +- **Database side**: Falls back to `supabase_service_role_key` in vault +- **Edge Function side**: Falls back to `SUPABASE_SERVICE_ROLE_KEY` env var + +Existing deployments will continue to work without immediate changes, though we strongly recommend migrating to `PGFLOW_AUTH_SECRET` to avoid the format mismatch issues.