From b6c00cf49eecfedbccb33e109d0a3dae9bafd8bb Mon Sep 17 00:00:00 2001 From: Artyom Keydunov Date: Fri, 23 Jan 2026 13:17:31 -0800 Subject: [PATCH 1/3] docs: creator mode (#10336) --- docs/pages/product/presentation/embedding.mdx | 2 + .../product/presentation/embedding/_meta.js | 3 +- .../presentation/embedding/creator-mode.mdx | 95 +++++++++++++++++++ .../embedding/signed-embedding.mdx | 10 ++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 docs/pages/product/presentation/embedding/creator-mode.mdx diff --git a/docs/pages/product/presentation/embedding.mdx b/docs/pages/product/presentation/embedding.mdx index 58d6e154e9dca..473e5e4b5878b 100644 --- a/docs/pages/product/presentation/embedding.mdx +++ b/docs/pages/product/presentation/embedding.mdx @@ -10,6 +10,8 @@ Cube supports iframe embedding for [dashboards](/product/presentation/dashboards - **[Signed Embedding](/product/presentation/embedding/signed-embedding)** – Authentication is handled programmatically using JWT tokens generated by your application. Users authenticate through your application and do not require Cube accounts. +- **[Creator Mode](/product/presentation/embedding/creator-mode)** – Embed the entire Cube application with workbooks-dashboard functionality, allowing users to create and modify dashboards directly within the embedded application. + Additionally, you can use the [Cube Core REST API](/product/apis-integrations/rest-api) directly for headless embedded analytics. ## Types of Embedding diff --git a/docs/pages/product/presentation/embedding/_meta.js b/docs/pages/product/presentation/embedding/_meta.js index fa486e0e4e056..c00b928899f72 100644 --- a/docs/pages/product/presentation/embedding/_meta.js +++ b/docs/pages/product/presentation/embedding/_meta.js @@ -1,4 +1,5 @@ module.exports = { "private-embedding": "Private embedding", - "signed-embedding": "Signed embedding" + "signed-embedding": "Signed embedding", + "creator-mode": "Creator mode" } diff --git a/docs/pages/product/presentation/embedding/creator-mode.mdx b/docs/pages/product/presentation/embedding/creator-mode.mdx new file mode 100644 index 0000000000000..d5dde8e6812d8 --- /dev/null +++ b/docs/pages/product/presentation/embedding/creator-mode.mdx @@ -0,0 +1,95 @@ +# Creator mode + + + Creator mode is currently in **private preview**. To enable this feature for your account, please contact [Cube support](https://cube.dev/contact). + + +Creator mode enables you to embed the entire Cube application with workbooks-dashboard functionality. Users will be able to create and modify their dashboards directly within the embedded application. + +## How it works + +In creator mode, you embed the full Cube application instead of individual dashboards or chat interfaces. This provides users with the complete Cube experience, including the ability to: + +- Create new dashboards +- Modify existing dashboards +- Access all workbook and dashboard functionality +- Build custom analytics experiences + +To enable creator mode, you need to pass `creatorMode: true`, `tenantId`, and `tenantName` to the [Generate Session API][ref-generate-session] when generating an embed session. The tenant ID is used to scope all content that the user creates to a specific tenant, while the tenant name is a human-readable friendly name displayed in the user interface. + +## Tenant ID and Name + +When using creator mode, you must provide both `tenantId` and `tenantName` parameters when generating a session: + +- **`tenantId`** – Used to scope all content (dashboards, workbooks, etc.) that users create within the embedded application to a specific tenant. This ensures proper data isolation in multi-tenant scenarios. +- **`tenantName`** – A human-readable friendly name for the tenant that will be displayed in the user interface. + +## Using creator mode + +To use creator mode and embed an app: + +1. **Set the embed type to "App"** in the form +2. **Fill in Deployment ID**, **Tenant ID**, **Tenant Name**, and either **External ID** or **Internal ID** (email) +3. **Click "Generate Session & Embed"** — the request automatically includes `creatorMode: true` for app embeddings +4. The app is embedded at `/embed/d/{deploymentId}/app?session={sessionId}` and displayed in the iframe + +Creator mode is enabled automatically when the embed type is "app"; no additional configuration is needed. The tenant ID is required to scope all dashboards and content created by the user to the specific tenant, and the tenant name will be displayed in the user interface. + +### Example + +```javascript +const API_KEY = "YOUR_API_KEY"; +const DEPLOYMENT_ID = 32; +const TENANT_ID = "tenant-123"; +const TENANT_NAME = "Acme Corporation"; + +const session = await fetch( + "https://your-account.cubecloud.dev/api/v1/embed/generate-session", + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Api-Key ${API_KEY}`, + }, + body: JSON.stringify({ + deploymentId: DEPLOYMENT_ID, + externalId: "user@example.com", + creatorMode: true, + tenantId: TENANT_ID, + tenantName: TENANT_NAME, + }), + }, +); + +const data = await session.json(); +const sessionId = data.sessionId; +``` + +### Embedding the app + +Use the session ID to embed the full Cube application: + +```html + +``` + +Replace `{deploymentId}` with your deployment ID and `{sessionId}` with the session ID returned from the Generate Session API. + +## Example application + +For a complete working example of embedding, including creator mode, check out the [cube-embedding-demo](https://github.com/cubedevinc/cube-embedding-demo) repository. This demo application provides: + +- A full working example of iframe embedding +- Implementation of signed iframe embedding with session generation +- Support for creator mode with tenant ID and tenant name +- A React-based UI for testing embedding functionality +- Backend server that securely handles API key authentication + +You can clone the repository, configure it with your Cube credentials, and run it locally to test embedding functionality or use it as a reference implementation for your own application. + +[ref-generate-session]: /product/apis-integrations/embed-apis/generate-session diff --git a/docs/pages/product/presentation/embedding/signed-embedding.mdx b/docs/pages/product/presentation/embedding/signed-embedding.mdx index a30a2a80e8825..4e672c3f5f448 100644 --- a/docs/pages/product/presentation/embedding/signed-embedding.mdx +++ b/docs/pages/product/presentation/embedding/signed-embedding.mdx @@ -175,6 +175,16 @@ User attributes enable row-level security and personalized chat responses by fil - Regional managers see data filtered by their city - Department heads see only their department's metrics +## Example application + +For a complete working example of signed embedding, check out the [cube-embedding-demo](https://github.com/cubedevinc/cube-embedding-demo) repository. This demo application provides: + +- A full working example of iframe embedding +- Implementation of signed iframe embedding with session generation +- A React-based UI for testing embedding functionality +- Backend server that securely handles API key authentication + +You can clone the repository, configure it with your Cube credentials, and run it locally to test embedding functionality or use it as a reference implementation for your own application. [ref-api-keys]: /product/administration/workspace/api-keys [ref-generate-session]: /product/apis-integrations/embed-apis/generate-session From 7f6049a17f68aa5bbcd63da2486b8ad74e4c7d4b Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Fri, 23 Jan 2026 19:32:00 -0500 Subject: [PATCH 2/3] add dd metrics export info docs (#10329) --- .../workspace/monitoring/datadog.mdx | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/pages/product/administration/workspace/monitoring/datadog.mdx b/docs/pages/product/administration/workspace/monitoring/datadog.mdx index 562afc14af75e..183c6a52c238e 100644 --- a/docs/pages/product/administration/workspace/monitoring/datadog.mdx +++ b/docs/pages/product/administration/workspace/monitoring/datadog.mdx @@ -1,7 +1,7 @@ # Integration with Datadog [Datadog][datadog] is a popular fully managed observability service. This guide -demonstrates how to set up Cube Cloud to export logs to Datadog. +demonstrates how to set up Cube Cloud to export logs and metrics to Datadog. ## Configuration @@ -45,9 +45,36 @@ navigate to Logs in Datadog and watch the logs coming: +### Exporting metrics + +To export metrics to Datadog, use the same API key from +Organization Settings → API Keys as configured for logs. + +Then, configure the [`datadog_metrics`][vector-datadog-metrics] sink in your +[`vector.toml` configuration file][ref-monitoring-integrations-conf]. + +Example configuration: + +```toml +[sinks.datadog_metrics] +type = "datadog_metrics" +inputs = [ + "metrics" +] +default_api_key = "$CUBE_CLOUD_MONITORING_DATADOG_API_KEY" +site = "datadoghq.eu" +``` + +Again, upon commit the configuration for Vector should take effect in a minute. Then, +navigate to Metrics → Summary in Datadog and explore the available +metrics. Cube metrics are prefixed with `cube_`, such as `cube_cpu_usage_ratio`, +`cube_memory_usage_ratio`, and `cube_requests_total`. + [datadog]: https://www.datadoghq.com [datadog-docs-sites]: https://docs.datadoghq.com/getting_started/site/ [vector-datadog-logs]: https://vector.dev/docs/reference/configuration/sinks/datadog_logs/ +[vector-datadog-metrics]: + https://vector.dev/docs/reference/configuration/sinks/datadog_metrics/ [ref-monitoring-integrations]: /product/workspace/monitoring [ref-monitoring-integrations-conf]: /product/workspace/monitoring#configuration From fe714bf333dec59a3988d626fdd709062a163b4f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Sat, 24 Jan 2026 02:49:23 +0200 Subject: [PATCH 3/3] feat(server-core): Exponential backoff in refresh scheduler (#10302) * add CUBEJS_PRE_AGG_BACKOFF_MAX_TIME env * implement PreAggBackoff api in orchestrator pre-aggs * impl backoff in refreshScheduler * wip: debug logs * add tests * Revert "wip: debug logs" This reverts commit 8f7bfb6928baee79ef0d199b2fb29a7146808791. * renaming * docs: New env var * . --------- Co-authored-by: Igor Lukanin --- .../caching/refreshing-pre-aggregations.mdx | 1 + .../reference/environment-variables.mdx | 11 ++ packages/cubejs-backend-shared/src/env.ts | 7 + .../src/orchestrator/PreAggregations.ts | 38 ++++++ .../src/core/RefreshScheduler.ts | 44 ++++++- .../test/unit/RefreshScheduler.test.ts | 124 +++++++++++++++++- 6 files changed, 222 insertions(+), 3 deletions(-) diff --git a/docs/pages/product/caching/refreshing-pre-aggregations.mdx b/docs/pages/product/caching/refreshing-pre-aggregations.mdx index 1e947f009331b..c6207695e4d5d 100644 --- a/docs/pages/product/caching/refreshing-pre-aggregations.mdx +++ b/docs/pages/product/caching/refreshing-pre-aggregations.mdx @@ -16,6 +16,7 @@ behavior: - `CUBEJS_REFRESH_WORKER_CONCURRENCY` (see also `CUBEJS_CONCURRENCY`) - `CUBEJS_SCHEDULED_REFRESH_QUERIES_PER_APP_ID` - `CUBEJS_DROP_PRE_AGG_WITHOUT_TOUCH` +- `CUBEJS_PRE_AGGREGATIONS_BACKOFF_MAX_TIME` ## Troubleshooting diff --git a/docs/pages/product/configuration/reference/environment-variables.mdx b/docs/pages/product/configuration/reference/environment-variables.mdx index be11ea3eb7ea8..ba110d1317945 100644 --- a/docs/pages/product/configuration/reference/environment-variables.mdx +++ b/docs/pages/product/configuration/reference/environment-variables.mdx @@ -1155,6 +1155,17 @@ This can be overridden for individual pre-aggregations using the | --------------- | ---------------------- | --------------------- | | `true`, `false` | `true` | `true` | +## `CUBEJS_PRE_AGGREGATIONS_BACKOFF_MAX_TIME` + +The maximum time, in seconds, for exponential backoff for retries when pre-aggregation +builds fail. When a pre-aggregation refresh fails, retries will be executed with +exponentially increasing delays, but the delay will not exceed the value specified by +this variable. + +| Possible Values | Default in Development | Default in Production | +| ------------------------- | ---------------------- | --------------------- | +| A valid number in seconds | `600` | `600` | + ## `CUBEJS_REFRESH_WORKER` If `true`, this instance of Cube will **only** refresh pre-aggregations. diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 327a5b14d3b2b..4781b0e3688d5 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -751,6 +751,13 @@ const variables: Record any> = { .default(60 * 60 * 24) .asIntPositive(), + /** + * Maximum time for exponential backoff for pre-aggs (in seconds) + */ + preAggBackoffMaxTime: (): number => get('CUBEJS_PRE_AGGREGATIONS_BACKOFF_MAX_TIME') + .default(10 * 60) + .asIntPositive(), + /** * Expire time for touch records */ diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts index 43604fc040234..cbf90c511e279 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts @@ -251,6 +251,8 @@ export class PreAggregations { private readonly touchTablePersistTime: number; + private readonly preAggBackoffMaxTime: number; + public readonly dropPreAggregationsWithoutTouch: boolean; private readonly usedTablePersistTime: number; @@ -277,6 +279,7 @@ export class PreAggregations { this.externalDriverFactory = options.externalDriverFactory; this.structureVersionPersistTime = options.structureVersionPersistTime || 60 * 60 * 24 * 30; this.touchTablePersistTime = options.touchTablePersistTime || getEnv('touchPreAggregationTimeout'); + this.preAggBackoffMaxTime = options.preAggBackoffMaxTime || getEnv('preAggBackoffMaxTime'); this.dropPreAggregationsWithoutTouch = options.dropPreAggregationsWithoutTouch || getEnv('dropPreAggregationsWithoutTouch'); this.usedTablePersistTime = options.usedTablePersistTime || getEnv('dbQueryTimeout'); this.externalRefresh = options.externalRefresh; @@ -317,6 +320,11 @@ export class PreAggregations { return this.queryCache.getKey('SQL_PRE_AGGREGATIONS_TABLES_TOUCH', tableName); } + protected preAggBackoffRedisKey(tableName: string): string { + // TODO add dataSource? + return this.queryCache.getKey('SQL_PRE_AGGREGATIONS_BACKOFF', tableName); + } + protected refreshEndReachedKey() { // TODO add dataSource? return this.queryCache.getKey('SQL_PRE_AGGREGATIONS_REFRESH_END_REACHED', ''); @@ -372,6 +380,36 @@ export class PreAggregations { .map(k => k.replace(this.tablesTouchRedisKey(''), '')); } + public async updatePreAggBackoff(tableName: string, backoffData: { backoffMultiplier: number, nextTimestamp: Date }): Promise { + await this.queryCache.getCacheDriver().set( + this.preAggBackoffRedisKey(tableName), + JSON.stringify(backoffData), + this.preAggBackoffMaxTime + ); + } + + public async removePreAggBackoff(tableName: string): Promise { + await this.queryCache.getCacheDriver().remove(this.preAggBackoffRedisKey(tableName)); + } + + public getPreAggBackoffMaxTime(): number { + return this.preAggBackoffMaxTime; + } + + public async getPreAggBackoff(tableName: string): Promise<{ backoffMultiplier: number, nextTimestamp: Date } | null> { + const res = await this.queryCache.getCacheDriver().get(this.preAggBackoffRedisKey(tableName)); + + if (!res) { + return null; + } + + const parsed = JSON.parse(res); + return { + backoffMultiplier: parsed.backoffMultiplier, + nextTimestamp: new Date(parsed.nextTimestamp), + }; + } + public async updateRefreshEndReached() { return this.queryCache.getCacheDriver().set(this.refreshEndReachedKey(), new Date().getTime(), this.touchTablePersistTime); } diff --git a/packages/cubejs-server-core/src/core/RefreshScheduler.ts b/packages/cubejs-server-core/src/core/RefreshScheduler.ts index 50b0d9b8dc924..e94001b9a6bb4 100644 --- a/packages/cubejs-server-core/src/core/RefreshScheduler.ts +++ b/packages/cubejs-server-core/src/core/RefreshScheduler.ts @@ -615,7 +615,49 @@ export class RefreshScheduler { const currentQuery = await queryIterator.current(); if (currentQuery && queryIterator.partitionCounter() % concurrency === workerIndex) { const orchestratorApi = await this.serverCore.getOrchestratorApi(context); - await orchestratorApi.executeQuery({ ...currentQuery, preAggregationsLoadCacheByDataSource }); + const preAggsInstance = orchestratorApi.getQueryOrchestrator().getPreAggregations(); + const now = new Date(); + + const backoffChecks = await Promise.all( + currentQuery.preAggregations.map(p => preAggsInstance.getPreAggBackoff(p.tableName)) + ); + + // Skip execution if any pre-aggregation is still in backoff window + const shouldSkip = backoffChecks.some(backoffData => backoffData && now < backoffData.nextTimestamp); + + if (!shouldSkip) { + try { + await orchestratorApi.executeQuery({ ...currentQuery, preAggregationsLoadCacheByDataSource }); + } catch (e: any) { + // Check if this is a "Continue wait" error - these are normal queue signals + // For Continue wait errors, re-throw to handle them in the normal flow + if (e.error === 'Continue wait') { + throw e; + } + + // Real datasource error - apply exponential backoff + for (const p of currentQuery.preAggregations) { + let backoffData = await preAggsInstance.getPreAggBackoff(p.tableName); + + if (backoffData && backoffData.backoffMultiplier > 0) { + const newMultiplier = backoffData.backoffMultiplier * 2; + const delaySeconds = Math.min(newMultiplier, preAggsInstance.getPreAggBackoffMaxTime()); + + backoffData = { + backoffMultiplier: newMultiplier, + nextTimestamp: new Date(now.valueOf() + delaySeconds * 1000), + }; + } else { + backoffData = { + backoffMultiplier: 1, + nextTimestamp: new Date(now.valueOf() + 1000), + }; + } + + await preAggsInstance.updatePreAggBackoff(p.tableName, backoffData); + } + } + } } const hasNext = await queryIterator.advance(); if (!hasNext) { diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 22309182253cc..8a274e0256f63 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -248,6 +248,12 @@ class MockDriver extends BaseDriver { private schema: any; + public shouldFailQuery: boolean = false; + + public failQueryPattern: RegExp | null = null; + + public queryAttempts: number = 0; + public constructor() { super(); } @@ -257,9 +263,22 @@ class MockDriver extends BaseDriver { public query(query) { this.executedQueries.push(query); + + // Track query attempts for backoff testing + if (this.failQueryPattern && query.match(this.failQueryPattern)) { + this.queryAttempts++; + } + let promise: any = Promise.resolve([query]); promise = promise.then((res) => new Promise(resolve => setTimeout(() => resolve(res), 150))); + // Simulate query failure for backoff testing + if (this.shouldFailQuery && this.failQueryPattern && query.match(this.failQueryPattern)) { + promise = promise.then(() => { + throw new Error('Simulated datasource error'); + }); + } + if (query.match(/min\(.*timestamp.*foo/)) { promise = promise.then(() => [{ min: '2020-12-27T00:00:00.000' }]); } @@ -331,7 +350,11 @@ class MockDriver extends BaseDriver { let testCounter = 1; -const setupScheduler = ({ repository, useOriginalSqlPreAggregations, skipAssertSecurityContext }: { repository: SchemaFileRepository, useOriginalSqlPreAggregations?: boolean, skipAssertSecurityContext?: true }) => { +const setupScheduler = ({ repository, useOriginalSqlPreAggregations, skipAssertSecurityContext }: { + repository: SchemaFileRepository, + useOriginalSqlPreAggregations?: boolean, + skipAssertSecurityContext?: true +}) => { const mockDriver = new MockDriver(); const externalDriver = new MockDriver(); @@ -362,7 +385,7 @@ const setupScheduler = ({ repository, useOriginalSqlPreAggregations, skipAssertS return externalDriver; }, orchestratorOptions: () => ({ - continueWaitTimeout: 0.1, + continueWaitTimeout: 1, queryCacheOptions: { queueOptions: () => ({ concurrency: 2, @@ -1112,4 +1135,101 @@ describe('Refresh Scheduler', () => { } } }); + + test('Exponential backoff', async () => { + process.env.CUBEJS_EXTERNAL_DEFAULT = 'false'; + process.env.CUBEJS_SCHEDULED_REFRESH_DEFAULT = 'true'; + process.env.CUBEJS_PRE_AGGREGATIONS_BACKOFF_MAX_TIME = '10'; // 10 seconds max backoff + + const { + refreshScheduler, mockDriver, serverCore + } = setupScheduler({ repository: repositoryWithPreAggregations }); + + const ctx = { authInfo: { tenantId: 'tenant1' }, securityContext: { tenantId: 'tenant1' }, requestId: 'XXX' }; + + const orchestratorApi = await serverCore.getOrchestratorApi(ctx); + const preAggsInstance = orchestratorApi.getQueryOrchestrator().getPreAggregations(); + + // Target specific pre-aggregation: foo_first (all partitions) + // Scheduler processes multiple partitions: foo_first20201231, foo_first20201230, etc. + // Configure driver to fail only for foo_first table creation + mockDriver.shouldFailQuery = true; + mockDriver.failQueryPattern = /foo_first/; + + // Run refresh until it tries to create foo_first table and fails + const queryIteratorState = {}; + const maxIterations = 100; + for (let i = 0; i < maxIterations; i++) { + try { + await refreshScheduler.runScheduledRefresh(ctx, { + concurrency: 1, + workerIndices: [0], + timezones: ['UTC'], + queryIteratorState, + }); + } catch (e) { + // Expected to fail when hitting foo_first + } + + // Check if we started attempting to create foo_first table + if (mockDriver.queryAttempts > 0) { + break; + } + } + + const initialAttempts = mockDriver.queryAttempts; + expect(initialAttempts).toBeGreaterThan(0); + + // Wait for backoff to be set in storage (increased delay for async Redis writes) + await mockDriver.delay(1000); + + // Find which foo_first partition has backoff set + // Scheduler may process different partitions (20201231, 20201230, etc.) + const possiblePartitions = ['20201231', '20201230', '20201229', '20201228', '20201227']; + let backoffData: { backoffMultiplier: number, nextTimestamp: Date } | null = null; + let targetTableName: string | null = null; + + for (const partition of possiblePartitions) { + const tableName = `stb_pre_aggregations.foo_first${partition}`; + const data = await preAggsInstance.getPreAggBackoff(tableName); + if (data) { + backoffData = data; + targetTableName = tableName; + break; + } + } + + // Verify backoff was set for at least one foo_first table + expect(backoffData).not.toBeNull(); + expect(targetTableName).not.toBeNull(); + // Initial backoff multiplier is 1 second + expect(backoffData!.backoffMultiplier).toBeGreaterThanOrEqual(1); + + // Step 1: Immediate retry - should skip due to backoff (10-second window) + const beforeSkipAttempts = mockDriver.queryAttempts; + const immediateRetryCount = 5; + for (let i = 0; i < immediateRetryCount; i++) { + try { + await refreshScheduler.runScheduledRefresh(ctx, { + concurrency: 1, + workerIndices: [0], + timezones: ['UTC'], + queryIteratorState, + }); + } catch (e) { + // Expected to skip due to backoff + } + } + + // Query attempts should not increase significantly (skipped due to backoff) + // Allow some margin for other pre-aggregations processed by scheduler + expect(mockDriver.queryAttempts).toBeLessThanOrEqual(beforeSkipAttempts + 2); + + // Step 2: Verify backoff persists - pre-aggregation is still in backoff after 500ms + await mockDriver.delay(500); + const backoffDataStillActive = await preAggsInstance.getPreAggBackoff(targetTableName!); + expect(backoffDataStillActive).not.toBeNull(); + // backoffDataStillActive exists, which means backoff is still in place + // (nextTimestamp may be close to current time due to test execution delays) + }); });