From df8161c4c1ad782493e83088c9260043996eb4c0 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Fri, 19 Dec 2025 15:01:39 +0100 Subject: [PATCH 1/2] feat: Allow to specify key for dimensions in schema (#10270) --- packages/cubejs-api-gateway/openspec.yml | 3 + packages/cubejs-client-core/src/types.ts | 4 +- .../src/compiler/CubeEvaluator.ts | 2 + .../src/compiler/CubeSymbols.ts | 79 +++++ .../src/compiler/CubeToMetaTransformer.ts | 3 + .../src/compiler/CubeValidator.ts | 2 + .../transpilers/CubePropContextTranspiler.ts | 2 +- .../unit/__snapshots__/views.test.ts.snap | 7 + .../test/unit/dimension-key.test.ts | 321 ++++++++++++++++++ .../src/models/v1_cube_meta_dimension.rs | 4 + 10 files changed, 424 insertions(+), 3 deletions(-) create mode 100644 packages/cubejs-schema-compiler/test/unit/dimension-key.test.ts diff --git a/packages/cubejs-api-gateway/openspec.yml b/packages/cubejs-api-gateway/openspec.yml index 136f15dfb731f..bb784655d044b 100644 --- a/packages/cubejs-api-gateway/openspec.yml +++ b/packages/cubejs-api-gateway/openspec.yml @@ -147,6 +147,9 @@ components: $ref: "#/components/schemas/V1CubeMetaFormat" order: $ref: "#/components/schemas/V1CubeMetaDimensionOrder" + key: + type: "string" + description: "Key reference for the dimension" V1CubeMetaDimensionOrder: type: "string" enum: diff --git a/packages/cubejs-client-core/src/types.ts b/packages/cubejs-client-core/src/types.ts index ca2dc2d9fb076..cfa1e9b045e98 100644 --- a/packages/cubejs-client-core/src/types.ts +++ b/packages/cubejs-client-core/src/types.ts @@ -1,6 +1,5 @@ import Meta from './Meta'; import { TimeDimensionGranularity } from './time'; -import { TransportOptions } from './HttpTransport'; export type QueryOrder = 'asc' | 'desc' | 'none'; @@ -27,7 +26,7 @@ export type Annotation = { shortTitle: string; type: string; meta?: any; - format?: DimensionFormat; + format?: DimensionFormat | MeasureFormat; drillMembers?: any[]; drillMembersGrouped?: any; granularity?: GranularityAnnotation; @@ -396,6 +395,7 @@ export type BaseCubeDimension = BaseCubeMember & { primaryKey?: boolean; suggestFilterValues: boolean; format?: DimensionFormat; + key?: string; }; export type CubeTimeDimension = BaseCubeDimension & diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 92d63aab79412..4d91f58abf453 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -37,6 +37,8 @@ export type DimensionDefinition = { multiStage?: boolean; shiftInterval?: string; order?: 'asc' | 'desc'; + key?: (...args: any[]) => ToString; + keyReference?: string; }; export type TimeShiftDefinition = { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index eb8d2e945d518..9d6c0f6aafe8b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -36,6 +36,8 @@ export type CubeSymbolDefinition = { timeShift?: TimeshiftDefinition[]; format?: string; order?: 'asc' | 'desc'; + key?: (...args: any[]) => ToString; + keyReference?: string; }; export type HierarchyDefinition = { @@ -275,9 +277,16 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface const sortedByDependency = R.pipe( R.sortBy((c: CubeDefinition) => !!c.isView), )(cubes); + for (const cube of sortedByDependency) { const splitViews: SplitViews = {}; + this.symbols[cube.name] = this.transform(cube.name, errorReporter.inContext(`${cube.name} cube`), splitViews); + + if (!cube.isView) { + this.evaluateDimensionKeys(this.getCubeDefinition(cube.name), errorReporter.inContext(`${cube.name} cube`)); + } + for (const viewName of Object.keys(splitViews)) { // TODO can we define it when cubeList is defined? this.cubeList.push(splitViews[viewName]); @@ -542,6 +551,45 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface }); } + private evaluateDimensionKeys(cube: CubeDefinition, errorReporter: ErrorReporter) { + const dimensions = cube.dimensions || {}; + + // eslint-disable-next-line no-restricted-syntax + for (const [dimensionName, dimension] of Object.entries(dimensions)) { + if (dimension.key) { + const keyReference = this.evaluateReference( + cube.name, + dimension.key, + `Dimension '${cube.name}.${dimensionName}' key` + ); + + const [refCubeName, refDimensionName] = keyReference.split('.'); + + if (refCubeName !== cube.name) { + errorReporter.error( + `Dimension '${cube.name}.${dimensionName}' has a key that references dimension '${keyReference}' ` + + 'from a different cube. Key must reference a dimension within the same cube.' + ); + } else if (!dimensions[refDimensionName]) { + errorReporter.error( + `Dimension '${cube.name}.${dimensionName}' references key dimension '${refDimensionName}' ` + + `which does not exist in cube '${cube.name}'.` + ); + } else { + const referencedDimension = dimensions[refDimensionName]; + if (referencedDimension.key) { + errorReporter.error( + `Dimension '${cube.name}.${dimensionName}' references '${keyReference}' as its key, ` + + `but '${keyReference}' already defines its own key. Nested keys are not allowed.` + ); + } else { + dimension.keyReference = keyReference; + } + } + } + } + } + protected transformPreAggregations(preAggregations: Object) { // eslint-disable-next-line no-restricted-syntax for (const preAggregation of Object.values(preAggregations)) { @@ -836,6 +884,23 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface return this.symbols[cubeName]?.cubeObj()?.[type]?.[memberName]; } + protected processKeyReferenceForView( + keyReference: string, + viewName: string, + viewAllMembers: ViewResolvedMember[], + dimensionName: string + ): { keyReference: string } { + const viewKeyMember = viewAllMembers.find(v => v.member === keyReference); + + if (!viewKeyMember) { + throw new UserError( + `Dimension '${dimensionName}' has key '${keyReference}' but the key dimension is not included in view '${viewName}'` + ); + } + + return { keyReference: `${viewName}.${viewKeyMember.name}` }; + } + protected generateIncludeMembers(members: any[], type: string, targetCube: CubeDefinitionExtended, viewAllMembers: ViewResolvedMember[]) { return members.map(memberRef => { const path = memberRef.member.split('.'); @@ -900,6 +965,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface format: memberRef.override?.format || resolvedMember.format, ...(resolvedMember.granularities ? { granularities: resolvedMember.granularities } : {}), ...(resolvedMember.multiStage && { multiStage: resolvedMember.multiStage }), + ...(resolvedMember.keyReference && this.processKeyReferenceForView(resolvedMember.keyReference, targetCube.name, viewAllMembers, memberRef.member)), }; } else if (type === 'segments') { memberDefinition = { @@ -994,6 +1060,19 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface return options.originalSorting ? references : R.sortBy(R.identity, references) as any; } + public evaluateReference( + cube: string, + referencesFn: (...args: Array) => ToString, + context: string + ): string { + const result = this.evaluateReferences(cube, referencesFn); + if (Array.isArray(result)) { + throw new UserError(`${context} must be a single reference, not an array`); + } + + return result; + } + public pathFromArray(array: string[]): string { return array.join('.'); } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 18edc54609e37..fc4ae556bfe0c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -39,6 +39,7 @@ export interface ExtendedCubeSymbolDefinition extends CubeSymbolDefinition { drillMemberReferences?: any; cumulative?: boolean; aggType?: string; + keyReference?: string; } interface ExtendedCubeDefinition extends CubeDefinitionExtended { @@ -98,6 +99,7 @@ export type DimensionConfig = { aliasMember?: string; granularities?: GranularityDefinition[]; order?: 'asc' | 'desc'; + key?: string; }; export type SegmentConfig = { @@ -276,6 +278,7 @@ export class CubeToMetaTransformer implements CompilerInterface { })) : undefined, order: extendedDimDef.order, + key: extendedDimDef.keyReference, }; }), segments: Object.entries(extendedCube.segments || {}).map((nameToSegment: [string, any]) => { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index d04bd9af513b5..36f564f9a93d9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -279,6 +279,8 @@ const BaseDimensionWithoutSubQuery = { }), meta: Joi.any(), order: Joi.string().valid('asc', 'desc'), + key: Joi.func(), + keyReference: Joi.string(), values: Joi.when('type', { is: 'switch', then: Joi.array().items(Joi.string()), diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 5c43c7e4248f1..5dbe42d9f9c10 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -20,7 +20,7 @@ export const transpiledFieldsPatterns: Array = [ /^measures\.[_a-zA-Z][_a-zA-Z0-9]*\.(timeShift|time_shift)\.[0-9]+\.(timeDimension|time_dimension)$/, /^measures\.[_a-zA-Z][_a-zA-Z0-9]*\.(reduceBy|reduce_by|groupBy|group_by|addGroupBy|add_group_by)$/, /^(measures|dimensions)\.[_a-zA-Z][_a-zA-Z0-9]*\.case\.switch$/, - /^dimensions\.[_a-zA-Z][_a-zA-Z0-9]*\.(reduceBy|reduce_by|groupBy|group_by|addGroupBy|add_group_by)$/, + /^dimensions\.[_a-zA-Z][_a-zA-Z0-9]*\.(reduceBy|reduce_by|groupBy|group_by|addGroupBy|add_group_by|key)$/, /^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.indexes\.[_a-zA-Z][_a-zA-Z0-9]*\.columns$/, /^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.(timeDimensionReference|timeDimension|time_dimension|segments|dimensions|measures|rollups|segmentReferences|dimensionReferences|measureReferences|rollupReferences)$/, /^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.(timeDimensions|time_dimensions)\.\d+\.dimension$/, diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/views.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/views.test.ts.snap index 381b44f36a20a..774dbb89a4833 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/views.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/views.test.ts.snap @@ -11,6 +11,7 @@ Object { "format": "imageUrl", "granularities": undefined, "isVisible": true, + "key": undefined, "meta": Object { "key": "Meta.key for CubeA.id", }, @@ -29,6 +30,7 @@ Object { "format": "imageUrl", "granularities": undefined, "isVisible": true, + "key": undefined, "meta": Object { "key": "Meta.key for CubeB.other_id", }, @@ -112,6 +114,7 @@ Object { "format": "imageUrl", "granularities": undefined, "isVisible": true, + "key": undefined, "meta": Object { "key": "Meta.key for CubeB.other_id", }, @@ -195,6 +198,7 @@ Object { "format": "imageUrl", "granularities": undefined, "isVisible": true, + "key": undefined, "meta": Object { "key": "Meta.key for CubeA.id", }, @@ -213,6 +217,7 @@ Object { "format": "imageUrl", "granularities": undefined, "isVisible": true, + "key": undefined, "meta": Object { "key": "Meta.key for CubeB.id", }, @@ -231,6 +236,7 @@ Object { "format": "imageUrl", "granularities": undefined, "isVisible": true, + "key": undefined, "meta": Object { "key": "Meta.key for CubeB.other_id", }, @@ -314,6 +320,7 @@ Object { "format": "imageUrl", "granularities": undefined, "isVisible": true, + "key": undefined, "meta": Object { "key": "Meta.key for CubeB.other_id", }, diff --git a/packages/cubejs-schema-compiler/test/unit/dimension-key.test.ts b/packages/cubejs-schema-compiler/test/unit/dimension-key.test.ts new file mode 100644 index 0000000000000..302b66dd5ad18 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/dimension-key.test.ts @@ -0,0 +1,321 @@ +import { prepareYamlCompiler } from './PrepareCompiler'; +import { createSchemaYaml } from './utils'; + +describe('Dimension key property', () => { + it('resolves key reference correctly', async () => { + const { compiler, cubeEvaluator, metaTransformer } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [{ + name: 'Products', + sql_table: 'products', + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'number', + primary_key: true, + }, + { + name: 'name', + sql: 'name', + type: 'string', + key: 'CUBE.id', + }, + ], + }], + }) + ); + + await compiler.compile(); + + const productsDef = cubeEvaluator.getCubeDefinition('Products') as any; + expect(productsDef.dimensions.name.keyReference).toBe('Products.id'); + + // Check meta API exposure + const productsMeta = metaTransformer.cubes + .map((def) => def.config) + .find((def) => def.name === 'Products'); + expect(productsMeta).toBeDefined(); + + const nameDimension = productsMeta?.dimensions.find( + (d) => d.name === 'Products.name' + ); + expect(nameDimension?.key).toBe('Products.id'); + + // id dimension should not have a key + const idDimension = productsMeta?.dimensions.find( + (d) => d.name === 'Products.id' + ); + expect(idDimension?.key).toBeUndefined(); + }); + + it('rejects cross-cube key references', async () => { + const { compiler } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [ + { + name: 'Products', + sql_table: 'products', + dimensions: [{ + name: 'id', + sql: 'id', + type: 'number', + primary_key: true, + }], + }, + { + name: 'Orders', + sql_table: 'orders', + dimensions: [{ + name: 'product_name', + sql: 'product_name', + type: 'string', + key: 'Products.id', + }], + }, + ], + }) + ); + + await expect(compiler.compile()).rejects.toThrow( + /key that references dimension.*from a different cube/ + ); + }); + + it('rejects nested keys', async () => { + const { compiler } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [{ + name: 'Products', + sql_table: 'products', + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'number', + primary_key: true, + }, + { + name: 'sku', + sql: 'sku', + type: 'string', + key: 'CUBE.id', + }, + { + name: 'name', + sql: 'name', + type: 'string', + key: 'CUBE.sku', + }, + ], + }], + }) + ); + + await expect(compiler.compile()).rejects.toThrow( + /Nested keys are not allowed/ + ); + }); + + it('rejects key reference to non-existent dimension', async () => { + const { compiler } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [{ + name: 'Products', + sql_table: 'products', + dimensions: [{ + name: 'name', + sql: 'name', + type: 'string', + key: 'CUBE.nonExistent', + }], + }], + }) + ); + + // The error comes from symbol resolution before our validation + await expect(compiler.compile()).rejects.toThrow( + /cannot be resolved/ + ); + }); + + it('key property is inherited in views', async () => { + const { compiler, cubeEvaluator, metaTransformer } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [{ + name: 'Products', + sql_table: 'products', + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'number', + primary_key: true, + }, + { + name: 'name', + sql: 'name', + type: 'string', + key: 'CUBE.id', + }, + ], + }], + views: [{ + name: 'ProductsView', + cubes: [{ + join_path: 'Products', + includes: ['id', 'name'], + }], + }], + }) + ); + + await compiler.compile(); + + // Check that key reference is transformed to view naming + const viewDef = cubeEvaluator.getCubeDefinition('ProductsView') as any; + expect(viewDef.dimensions.name.keyReference).toBe('ProductsView.id'); + + // Check meta API + const viewMeta = metaTransformer.cubes + .map((def) => def.config) + .find((def) => def.name === 'ProductsView'); + expect(viewMeta).toBeDefined(); + + const nameDim = viewMeta?.dimensions.find( + (d) => d.name === 'ProductsView.name' + ); + expect(nameDim?.key).toBe('ProductsView.id'); + }); + + it('key property with prefixed includes', async () => { + const { compiler, cubeEvaluator, metaTransformer } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [{ + name: 'Products', + sql_table: 'products', + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'number', + primary_key: true, + }, + { + name: 'name', + sql: 'name', + type: 'string', + key: 'CUBE.id', + }, + ], + }], + views: [{ + name: 'AllDataView', + cubes: [{ + join_path: 'Products', + includes: '*', + prefix: true, + }], + }], + }) + ); + + await compiler.compile(); + + const viewDef = cubeEvaluator.getCubeDefinition('AllDataView') as any; + expect(viewDef.dimensions.Products_name.keyReference).toBe('AllDataView.Products_id'); + + const viewMeta = metaTransformer.cubes + .map((def) => def.config) + .find((def) => def.name === 'AllDataView'); + const nameDim = viewMeta?.dimensions.find( + (d) => d.name === 'AllDataView.Products_name' + ); + expect(nameDim?.key).toBe('AllDataView.Products_id'); + }); + + it('key property with aliased key dimension', async () => { + const { compiler, cubeEvaluator, metaTransformer } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [{ + name: 'Products', + sql_table: 'products', + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'number', + primary_key: true, + }, + { + name: 'name', + sql: 'name', + type: 'string', + key: 'CUBE.id', + }, + ], + }], + views: [{ + name: 'ProductsView', + cubes: [{ + join_path: 'Products', + includes: [ + { name: 'id', alias: 'product_id' }, + 'name', + ], + }], + }], + }) + ); + + await compiler.compile(); + + const viewDef = cubeEvaluator.getCubeDefinition('ProductsView') as any; + expect(viewDef.dimensions.name.keyReference).toBe('ProductsView.product_id'); + + const viewMeta = metaTransformer.cubes + .map((def) => def.config) + .find((def) => def.name === 'ProductsView'); + const nameDim = viewMeta?.dimensions.find( + (d) => d.name === 'ProductsView.name' + ); + expect(nameDim?.key).toBe('ProductsView.product_id'); + }); + + it('rejects view that excludes key dimension', async () => { + const { compiler } = prepareYamlCompiler( + createSchemaYaml({ + cubes: [{ + name: 'Products', + sql_table: 'products', + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'number', + primary_key: true, + }, + { + name: 'name', + sql: 'name', + type: 'string', + key: 'CUBE.id', + }, + ], + }], + views: [{ + name: 'ProductsView', + cubes: [{ + join_path: 'Products', + includes: '*', + excludes: ['id'], + }], + }], + }) + ); + + await expect(compiler.compile()).rejects.toThrow( + /key dimension is not included in view/ + ); + }); +}); diff --git a/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs b/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs index 4fde577c8f8c5..a931976b20686 100644 --- a/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs +++ b/rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs @@ -34,6 +34,9 @@ pub struct V1CubeMetaDimension { pub format: Option>, #[serde(rename = "order", skip_serializing_if = "Option::is_none")] pub order: Option, + /// Key reference for the dimension + #[serde(rename = "key", skip_serializing_if = "Option::is_none")] + pub key: Option, } impl V1CubeMetaDimension { @@ -49,6 +52,7 @@ impl V1CubeMetaDimension { meta: None, format: None, order: None, + key: None, } } } From 4e80f7bf4ecd534a58b2168023dc1464e35d739e Mon Sep 17 00:00:00 2001 From: waralexrom <108349432+waralexrom@users.noreply.github.com> Date: Fri, 19 Dec 2025 19:07:46 +0100 Subject: [PATCH 2/2] chore(cubestore): Temporary turn off some tests (#10273) --- packages/cubejs-testing-drivers/fixtures/redshift.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/cubejs-testing-drivers/fixtures/redshift.json b/packages/cubejs-testing-drivers/fixtures/redshift.json index ac1c913d8fbc9..a755b0fe2005d 100644 --- a/packages/cubejs-testing-drivers/fixtures/redshift.json +++ b/packages/cubejs-testing-drivers/fixtures/redshift.json @@ -191,6 +191,12 @@ "querying SwitchSourceTest: filter by switch dimensions", "querying BigECommerce: SeveralMultiStageMeasures", + "---------------------------------------", + "FIXME: temporarily disabled — doesn't work with cubestore-df46 ", + "---------------------------------------", + "querying ECommerce: partitioned pre-agg higher granularity", + "querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + dimension", + "---------------------------------------", "SKIPPED SQL API (Need work) ", "---------------------------------------",