From cc4665dfc64de28e779f042b149c952f0e8a059e Mon Sep 17 00:00:00 2001 From: jeffhiltz Date: Fri, 19 Dec 2025 11:48:38 -0500 Subject: [PATCH 1/6] initial version of Iceberg-backed Glue Table --- lib/shortcuts/api.md | 25 ++++ lib/shortcuts/glue-iceberg-table.js | 106 +++++++++++++++ lib/shortcuts/index.js | 1 + .../glue-iceberg-table-defaults.json | 50 +++++++ .../glue-iceberg-table-no-defaults.json | 60 +++++++++ ...e-iceberg-table-with-optimizer-custom.json | 93 +++++++++++++ ...iceberg-table-with-optimizer-defaults.json | 73 ++++++++++ test/shortcuts.test.js | 126 ++++++++++++++++++ 8 files changed, 534 insertions(+) create mode 100644 lib/shortcuts/glue-iceberg-table.js create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-defaults.json create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json diff --git a/lib/shortcuts/api.md b/lib/shortcuts/api.md index 8f93a2b..ff8ad89 100644 --- a/lib/shortcuts/api.md +++ b/lib/shortcuts/api.md @@ -12,6 +12,9 @@ a Lambda permission.

GlueDatabase

Create a Glue Database.

+
GlueIcebergTable
+

Create a Glue table backed by Apache Iceberg format on S3.

+
GlueJsonTable

Create a Glue Table backed by line-delimited JSON files on S3.

@@ -202,6 +205,28 @@ const db = new cf.shortcuts.GlueDatabase({ module.exports = cf.merge(myTemplate, db); ``` + + +## GlueIcebergTable +Create a Glue table backed by Apache Iceberg format on S3. + +**Kind**: global class + + +### new GlueIcebergTable(options) + +| Param | Type | Default | Description | +| --- | --- | --- | --- | +| options | Object | | Accepts the same options as cloudfriend's [`GlueTable`](https://github.com/mapbox/cloudfriend/blob/master/lib/shortcuts/glue-table.js), though the following additional attributes are either required or hard-wired: | +| options.Location | String | | The physical location of the table. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-storagedescriptor.html#cfn-glue-table-storagedescriptor-location). | +| [options.TableType] | String | 'EXTERNAL_TABLE' | Hard-wired by this shortcut. | +| [options.IcebergVersion] | String | '2' | The table version for the Iceberg table. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-iceberginput.html). | +| [options.EnableOptimizer] | Boolean | false | Whether to enable the snapshot retention optimizer for this Iceberg table. | +| [options.OptimizerRoleArn] | String | | The ARN of the IAM role for the table optimizer to use. Required if EnableOptimizer is true. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.SnapshotRetentionPeriodInDays] | Number | 5 | The number of days to retain snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | +| [options.NumberOfSnapshotsToRetain] | Number | 1 | The minimum number of snapshots to retain. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | +| [options.CleanExpiredFiles] | Boolean | true | Whether to delete expired data files after expiring snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | + ## GlueJsonTable diff --git a/lib/shortcuts/glue-iceberg-table.js b/lib/shortcuts/glue-iceberg-table.js new file mode 100644 index 0000000..0fec1f1 --- /dev/null +++ b/lib/shortcuts/glue-iceberg-table.js @@ -0,0 +1,106 @@ +'use strict'; + +const GlueTable = require('./glue-table'); + +/** + * Create a Glue table backed by Apache Iceberg format on S3. + * + * @param {Object} options - Accepts the same options as cloudfriend's + * [`GlueTable`](https://github.com/mapbox/cloudfriend/blob/master/lib/shortcuts/glue-table.js), + * though the following additional attributes are either required or hard-wired: + * @param {String} options.Location - The physical location of the table. See + * [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-storagedescriptor.html#cfn-glue-table-storagedescriptor-location). + * @param {String} [options.TableType='EXTERNAL_TABLE'] - Hard-wired by this + * shortcut. + * @param {String} [options.IcebergVersion='2'] - The table version for the + * Iceberg table. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-iceberginput.html). + * @param {Boolean} [options.EnableOptimizer=false] - Whether to enable the + * snapshot retention optimizer for this Iceberg table. + * @param {String} [options.OptimizerRoleArn=undefined] - The ARN of the IAM + * role for the table optimizer to use. Required if EnableOptimizer is true. + * See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). + * @param {Number} [options.SnapshotRetentionPeriodInDays=5] - The number of + * days to retain snapshots. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). + * @param {Number} [options.NumberOfSnapshotsToRetain=1] - The minimum number + * of snapshots to retain. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). + * @param {Boolean} [options.CleanExpiredFiles=true] - Whether to delete + * expired data files after expiring snapshots. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). + */ +class GlueIcebergTable extends GlueTable { + constructor(options) { + if (!options) throw new Error('Options required'); + const { + Location, + IcebergVersion = '2', + EnableOptimizer = false, + OptimizerRoleArn, + SnapshotRetentionPeriodInDays = 5, + NumberOfSnapshotsToRetain = 1, + CleanExpiredFiles = true + } = options; + + const required = [Location]; + if (required.some((variable) => !variable)) + throw new Error('You must provide a Location'); + + if (EnableOptimizer && !OptimizerRoleArn) + throw new Error('You must provide an OptimizerRoleArn when EnableOptimizer is true'); + + super( + Object.assign( + { + TableType: 'EXTERNAL_TABLE', + Parameters: { EXTERNAL: 'TRUE' } + }, + options + ) + ); + + const logicalName = options.LogicalName; + this.Resources[logicalName].Properties.OpenTableFormatInput = { + IcebergInput: { + MetadataOperation: 'CREATE', + Version: IcebergVersion + } + }; + + // Optionally add TableOptimizer for configuring snapshot retention + if (EnableOptimizer) { + const optimizerLogicalName = `${logicalName}Optimizer`; + this.Resources[optimizerLogicalName] = { + Type: 'AWS::Glue::TableOptimizer', + DependsOn: logicalName, + Properties: { + CatalogId: options.CatalogId || { Ref: 'AWS::AccountId' }, + DatabaseName: options.DatabaseName, + TableName: options.Name, + Type: 'retention', + TableOptimizerConfiguration: { + RoleArn: OptimizerRoleArn, + Enabled: true, + RetentionConfiguration: { + IcebergConfiguration: { + SnapshotRetentionPeriodInDays, + NumberOfSnapshotsToRetain, + CleanExpiredFiles + } + } + } + } + }; + + // Apply Condition to optimizer if specified on the table + if (options.Condition) { + this.Resources[optimizerLogicalName].Condition = options.Condition; + } + } + } +} + +module.exports = GlueIcebergTable; diff --git a/lib/shortcuts/index.js b/lib/shortcuts/index.js index 54e40fe..63fa0f9 100644 --- a/lib/shortcuts/index.js +++ b/lib/shortcuts/index.js @@ -16,6 +16,7 @@ module.exports = { GlueJsonTable: require('./glue-json-table'), GlueOrcTable: require('./glue-orc-table'), GlueParquetTable: require('./glue-parquet-table'), + GlueIcebergTable: require('./glue-iceberg-table'), GluePrestoView: require('./glue-presto-view'), GlueSparkView: require('./glue-spark-view'), hookshot: require('./hookshot'), diff --git a/test/fixtures/shortcuts/glue-iceberg-table-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-defaults.json new file mode 100644 index 0000000..2d00ccf --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-defaults.json @@ -0,0 +1,50 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json new file mode 100644 index 0000000..3c16713 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json @@ -0,0 +1,60 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": { + "Always": { + "Fn::Equals": [ + "1", + "1" + ] + } + }, + "Resources": { + "AnotherThing": { + "Type": "AWS::SNS::Topic" + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Condition": "Always", + "DependsOn": "AnotherThing", + "Properties": { + "CatalogId": "1234", + "DatabaseName": "my_database", + "TableInput": { + "Description": "my_table description", + "Name": "my_table", + "Owner": "Team", + "Parameters": { + "table": "params" + }, + "PartitionKeys": [], + "Retention": 12, + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json new file mode 100644 index 0000000..29962ca --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json @@ -0,0 +1,93 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": { + "Always": { + "Fn::Equals": [ + "1", + "1" + ] + } + }, + "Resources": { + "OptimizerRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": {} + } + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Condition": "Always", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": { + "Fn::GetAtt": [ + "OptimizerRole", + "Arn" + ] + }, + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 7, + "NumberOfSnapshotsToRetain": 3, + "CleanExpiredFiles": false + } + } + } + }, + "Condition": "Always" + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json new file mode 100644 index 0000000..f21a4a8 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json @@ -0,0 +1,73 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/OptimizerRole", + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 5, + "NumberOfSnapshotsToRetain": 1, + "CleanExpiredFiles": true + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/shortcuts.test.js b/test/shortcuts.test.js index 85f222d..79ce22a 100644 --- a/test/shortcuts.test.js +++ b/test/shortcuts.test.js @@ -1422,6 +1422,132 @@ test('[shortcuts] glue parquet table', (assert) => { assert.end(); }); +test('[shortcuts] glue iceberg table', (assert) => { + assert.throws( + () => new cf.shortcuts.GlueIcebergTable(), + 'Options required', + 'throws without options' + ); + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({}), + /You must provide a Location/, + 'throws without required parameters' + ); + + let db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location' + }); + + let template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-defaults'), + 'expected resources generated with defaults' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + CatalogId: '1234', + Owner: 'Team', + Parameters: { table: 'params' }, + Description: 'my_table description', + Retention: 12, + Location: 's3://fake/location', + IcebergVersion: '2', + Condition: 'Always', + DependsOn: 'AnotherThing' + }); + + template = cf.merge( + { Conditions: { Always: cf.equals('1', '1') } }, + { Resources: { AnotherThing: { Type: 'AWS::SNS::Topic' } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-no-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-no-defaults'), + 'expected resources generated without defaults' + ); + + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true + }), + /You must provide an OptimizerRoleArn when EnableOptimizer is true/, + 'throws when EnableOptimizer is true but OptimizerRoleArn is missing' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: 'arn:aws:iam::123456789012:role/OptimizerRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-optimizer-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-optimizer-defaults'), + 'expected resources generated with optimizer using default retention settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: cf.getAtt('OptimizerRole', 'Arn'), + SnapshotRetentionPeriodInDays: 7, + NumberOfSnapshotsToRetain: 3, + CleanExpiredFiles: false, + Condition: 'Always' + }); + + template = cf.merge( + { Conditions: { Always: cf.equals('1', '1') } }, + { Resources: { OptimizerRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-with-optimizer-custom', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-optimizer-custom'), + 'expected resources generated with optimizer using custom retention settings' + ); + + assert.end(); +}); + test('[shortcuts] glue view', (assert) => { assert.throws( () => new cf.shortcuts.GluePrestoView(), From 95dcd927861ca4b176abbedf5df0c28c2176afd5 Mon Sep 17 00:00:00 2001 From: jeffhiltz Date: Fri, 19 Dec 2025 12:30:41 -0500 Subject: [PATCH 2/6] add compaction configuration to Glue Iceberg tables --- lib/shortcuts/api.md | 7 +- lib/shortcuts/glue-iceberg-table.js | 66 +++++++++++- ...ue-iceberg-table-with-both-optimizers.json | 96 +++++++++++++++++ ...-iceberg-table-with-compaction-custom.json | 84 +++++++++++++++ ...ceberg-table-with-compaction-defaults.json | 73 +++++++++++++ ...e-iceberg-table-with-optimizer-custom.json | 2 +- ...iceberg-table-with-optimizer-defaults.json | 2 +- test/shortcuts.test.js | 101 ++++++++++++++++++ 8 files changed, 424 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json diff --git a/lib/shortcuts/api.md b/lib/shortcuts/api.md index ff8ad89..580058b 100644 --- a/lib/shortcuts/api.md +++ b/lib/shortcuts/api.md @@ -222,10 +222,15 @@ Create a Glue table backed by Apache Iceberg format on S3. | [options.TableType] | String | 'EXTERNAL_TABLE' | Hard-wired by this shortcut. | | [options.IcebergVersion] | String | '2' | The table version for the Iceberg table. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-iceberginput.html). | | [options.EnableOptimizer] | Boolean | false | Whether to enable the snapshot retention optimizer for this Iceberg table. | -| [options.OptimizerRoleArn] | String | | The ARN of the IAM role for the table optimizer to use. Required if EnableOptimizer is true. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.OptimizerRoleArn] | String | | The ARN of the IAM role for the retention optimizer to use. Required if EnableOptimizer is true. Can be the same role as CompactionRoleArn if both optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | | [options.SnapshotRetentionPeriodInDays] | Number | 5 | The number of days to retain snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | | [options.NumberOfSnapshotsToRetain] | Number | 1 | The minimum number of snapshots to retain. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | | [options.CleanExpiredFiles] | Boolean | true | Whether to delete expired data files after expiring snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | +| [options.EnableCompaction] | Boolean | false | Whether to enable the compaction optimizer for this Iceberg table. | +| [options.CompactionRoleArn] | String | | The ARN of the IAM role for the compaction optimizer to use. Required if EnableCompaction is true. Can be the same role as OptimizerRoleArn if both optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.CompactionStrategy] | String | 'binpack' | The compaction strategy: binpack, sort, or z-order. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-compaction.html). | +| [options.MinInputFiles] | Number | 100 | Minimum number of data files before compaction triggers. | +| [options.DeleteFileThreshold] | Number | 1 | Minimum deletes in a file to make it eligible for compaction. | diff --git a/lib/shortcuts/glue-iceberg-table.js b/lib/shortcuts/glue-iceberg-table.js index 0fec1f1..3e84856 100644 --- a/lib/shortcuts/glue-iceberg-table.js +++ b/lib/shortcuts/glue-iceberg-table.js @@ -19,8 +19,9 @@ const GlueTable = require('./glue-table'); * @param {Boolean} [options.EnableOptimizer=false] - Whether to enable the * snapshot retention optimizer for this Iceberg table. * @param {String} [options.OptimizerRoleArn=undefined] - The ARN of the IAM - * role for the table optimizer to use. Required if EnableOptimizer is true. - * See [AWS + * role for the retention optimizer to use. Required if EnableOptimizer is + * true. Can be the same role as CompactionRoleArn if both optimizers are + * enabled. See [AWS * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). * @param {Number} [options.SnapshotRetentionPeriodInDays=5] - The number of * days to retain snapshots. See [AWS @@ -31,6 +32,20 @@ const GlueTable = require('./glue-table'); * @param {Boolean} [options.CleanExpiredFiles=true] - Whether to delete * expired data files after expiring snapshots. See [AWS * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). + * @param {Boolean} [options.EnableCompaction=false] - Whether to enable the + * compaction optimizer for this Iceberg table. + * @param {String} [options.CompactionRoleArn=undefined] - The ARN of the IAM + * role for the compaction optimizer to use. Required if EnableCompaction is + * true. Can be the same role as OptimizerRoleArn if both optimizers are + * enabled. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). + * @param {String} [options.CompactionStrategy='binpack'] - The compaction + * strategy: binpack, sort, or z-order. See [AWS + * documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-compaction.html). + * @param {Number} [options.MinInputFiles=100] - Minimum number of data files + * before compaction triggers. + * @param {Number} [options.DeleteFileThreshold=1] - Minimum deletes in a file + * to make it eligible for compaction. */ class GlueIcebergTable extends GlueTable { constructor(options) { @@ -42,7 +57,12 @@ class GlueIcebergTable extends GlueTable { OptimizerRoleArn, SnapshotRetentionPeriodInDays = 5, NumberOfSnapshotsToRetain = 1, - CleanExpiredFiles = true + CleanExpiredFiles = true, + EnableCompaction = false, + CompactionRoleArn, + CompactionStrategy = 'binpack', + MinInputFiles = 100, + DeleteFileThreshold = 1 } = options; const required = [Location]; @@ -52,6 +72,13 @@ class GlueIcebergTable extends GlueTable { if (EnableOptimizer && !OptimizerRoleArn) throw new Error('You must provide an OptimizerRoleArn when EnableOptimizer is true'); + if (EnableCompaction && !CompactionRoleArn) + throw new Error('You must provide a CompactionRoleArn when EnableCompaction is true'); + + const validStrategies = ['binpack', 'sort', 'z-order']; + if (!validStrategies.includes(CompactionStrategy)) + throw new Error('CompactionStrategy must be one of: binpack, sort, z-order'); + super( Object.assign( { @@ -72,7 +99,7 @@ class GlueIcebergTable extends GlueTable { // Optionally add TableOptimizer for configuring snapshot retention if (EnableOptimizer) { - const optimizerLogicalName = `${logicalName}Optimizer`; + const optimizerLogicalName = `${logicalName}RetentionOptimizer`; this.Resources[optimizerLogicalName] = { Type: 'AWS::Glue::TableOptimizer', DependsOn: logicalName, @@ -100,6 +127,37 @@ class GlueIcebergTable extends GlueTable { this.Resources[optimizerLogicalName].Condition = options.Condition; } } + + // Optionally add TableOptimizer for compaction + if (EnableCompaction) { + const compactionLogicalName = `${logicalName}CompactionOptimizer`; + this.Resources[compactionLogicalName] = { + Type: 'AWS::Glue::TableOptimizer', + DependsOn: logicalName, + Properties: { + CatalogId: options.CatalogId || { Ref: 'AWS::AccountId' }, + DatabaseName: options.DatabaseName, + TableName: options.Name, + Type: 'compaction', + TableOptimizerConfiguration: { + RoleArn: CompactionRoleArn, + Enabled: true, + CompactionConfiguration: { + IcebergConfiguration: { + Strategy: CompactionStrategy, + MinInputFiles, + DeleteFileThreshold + } + } + } + } + }; + + // Apply Condition to compaction optimizer if specified on the table + if (options.Condition) { + this.Resources[compactionLogicalName].Condition = options.Condition; + } + } } } diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json b/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json new file mode 100644 index 0000000..5874a2c --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json @@ -0,0 +1,96 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableRetentionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/RetentionRole", + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 5, + "NumberOfSnapshotsToRetain": 1, + "CleanExpiredFiles": true + } + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/CompactionRole", + "Enabled": true, + "CompactionConfiguration": { + "IcebergConfiguration": { + "Strategy": "sort", + "MinInputFiles": 100, + "DeleteFileThreshold": 1 + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json new file mode 100644 index 0000000..abe9583 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json @@ -0,0 +1,84 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "CompactionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": {} + } + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": { + "Fn::GetAtt": [ + "CompactionRole", + "Arn" + ] + }, + "Enabled": true, + "CompactionConfiguration": { + "IcebergConfiguration": { + "Strategy": "z-order", + "MinInputFiles": 50, + "DeleteFileThreshold": 10 + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json new file mode 100644 index 0000000..79476b1 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json @@ -0,0 +1,73 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/CompactionRole", + "Enabled": true, + "CompactionConfiguration": { + "IcebergConfiguration": { + "Strategy": "binpack", + "MinInputFiles": 100, + "DeleteFileThreshold": 1 + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json index 29962ca..cc3ed38 100644 --- a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json @@ -59,7 +59,7 @@ } } }, - "MyTableOptimizer": { + "MyTableRetentionOptimizer": { "Type": "AWS::Glue::TableOptimizer", "DependsOn": "MyTable", "Properties": { diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json index f21a4a8..742cb3f 100644 --- a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json @@ -45,7 +45,7 @@ } } }, - "MyTableOptimizer": { + "MyTableRetentionOptimizer": { "Type": "AWS::Glue::TableOptimizer", "DependsOn": "MyTable", "Properties": { diff --git a/test/shortcuts.test.js b/test/shortcuts.test.js index 79ce22a..857f683 100644 --- a/test/shortcuts.test.js +++ b/test/shortcuts.test.js @@ -1545,6 +1545,107 @@ test('[shortcuts] glue iceberg table', (assert) => { 'expected resources generated with optimizer using custom retention settings' ); + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableCompaction: true + }), + /You must provide a CompactionRoleArn when EnableCompaction is true/, + 'throws when EnableCompaction is true but CompactionRoleArn is missing' + ); + + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableCompaction: true, + CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole', + CompactionStrategy: 'invalid' + }), + /CompactionStrategy must be one of: binpack, sort, z-order/, + 'throws when CompactionStrategy is invalid' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableCompaction: true, + CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-compaction-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-compaction-defaults'), + 'expected resources generated with compaction using default settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableCompaction: true, + CompactionRoleArn: cf.getAtt('CompactionRole', 'Arn'), + CompactionStrategy: 'z-order', + MinInputFiles: 50, + DeleteFileThreshold: 10 + }); + + template = cf.merge( + { Resources: { CompactionRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-with-compaction-custom', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-compaction-custom'), + 'expected resources generated with compaction using custom settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: 'arn:aws:iam::123456789012:role/RetentionRole', + EnableCompaction: true, + CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole', + CompactionStrategy: 'sort' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-both-optimizers', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-both-optimizers'), + 'expected resources generated with both retention and compaction optimizers' + ); + assert.end(); }); From d84839d95a0043399e66e9d99b15918cc2e43100 Mon Sep 17 00:00:00 2001 From: jeffhiltz Date: Fri, 19 Dec 2025 14:45:25 -0500 Subject: [PATCH 3/6] add orphan file deletion configuration to Glue Iceberg tables --- lib/shortcuts/api.md | 8 +- lib/shortcuts/glue-iceberg-table.js | 67 +++++++++- ...lue-iceberg-table-with-all-optimizers.json | 117 ++++++++++++++++++ ...erg-table-with-orphan-deletion-custom.json | 83 +++++++++++++ ...g-table-with-orphan-deletion-defaults.json | 71 +++++++++++ test/shortcuts.test.js | 84 +++++++++++++ 6 files changed, 423 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json create mode 100644 test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json diff --git a/lib/shortcuts/api.md b/lib/shortcuts/api.md index 580058b..db38698 100644 --- a/lib/shortcuts/api.md +++ b/lib/shortcuts/api.md @@ -222,15 +222,19 @@ Create a Glue table backed by Apache Iceberg format on S3. | [options.TableType] | String | 'EXTERNAL_TABLE' | Hard-wired by this shortcut. | | [options.IcebergVersion] | String | '2' | The table version for the Iceberg table. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-iceberginput.html). | | [options.EnableOptimizer] | Boolean | false | Whether to enable the snapshot retention optimizer for this Iceberg table. | -| [options.OptimizerRoleArn] | String | | The ARN of the IAM role for the retention optimizer to use. Required if EnableOptimizer is true. Can be the same role as CompactionRoleArn if both optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.OptimizerRoleArn] | String | | The ARN of the IAM role for the retention optimizer to use. Required if EnableOptimizer is true. Can be the same role as CompactionRoleArn or OrphanFileDeletionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | | [options.SnapshotRetentionPeriodInDays] | Number | 5 | The number of days to retain snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | | [options.NumberOfSnapshotsToRetain] | Number | 1 | The minimum number of snapshots to retain. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | | [options.CleanExpiredFiles] | Boolean | true | Whether to delete expired data files after expiring snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | | [options.EnableCompaction] | Boolean | false | Whether to enable the compaction optimizer for this Iceberg table. | -| [options.CompactionRoleArn] | String | | The ARN of the IAM role for the compaction optimizer to use. Required if EnableCompaction is true. Can be the same role as OptimizerRoleArn if both optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.CompactionRoleArn] | String | | The ARN of the IAM role for the compaction optimizer to use. Required if EnableCompaction is true. Can be the same role as OptimizerRoleArn or OrphanFileDeletionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | | [options.CompactionStrategy] | String | 'binpack' | The compaction strategy: binpack, sort, or z-order. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-compaction.html). | | [options.MinInputFiles] | Number | 100 | Minimum number of data files before compaction triggers. | | [options.DeleteFileThreshold] | Number | 1 | Minimum deletes in a file to make it eligible for compaction. | +| [options.EnableOrphanFileDeletion] | Boolean | false | Whether to enable the orphan file deletion optimizer for this Iceberg table. | +| [options.OrphanFileDeletionRoleArn] | String | | The ARN of the IAM role for the orphan file deletion optimizer to use. Required if EnableOrphanFileDeletion is true. Can be the same role as OptimizerRoleArn or CompactionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.OrphanFileRetentionPeriodInDays] | Number | 3 | The number of days to retain orphan files before deleting them. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). | +| [options.OrphanFileDeletionLocation] | String | | The S3 location to scan for orphan files. Defaults to the table location if not specified. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). | diff --git a/lib/shortcuts/glue-iceberg-table.js b/lib/shortcuts/glue-iceberg-table.js index 3e84856..d9ccf05 100644 --- a/lib/shortcuts/glue-iceberg-table.js +++ b/lib/shortcuts/glue-iceberg-table.js @@ -20,8 +20,8 @@ const GlueTable = require('./glue-table'); * snapshot retention optimizer for this Iceberg table. * @param {String} [options.OptimizerRoleArn=undefined] - The ARN of the IAM * role for the retention optimizer to use. Required if EnableOptimizer is - * true. Can be the same role as CompactionRoleArn if both optimizers are - * enabled. See [AWS + * true. Can be the same role as CompactionRoleArn or OrphanFileDeletionRoleArn + * if multiple optimizers are enabled. See [AWS * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). * @param {Number} [options.SnapshotRetentionPeriodInDays=5] - The number of * days to retain snapshots. See [AWS @@ -36,8 +36,8 @@ const GlueTable = require('./glue-table'); * compaction optimizer for this Iceberg table. * @param {String} [options.CompactionRoleArn=undefined] - The ARN of the IAM * role for the compaction optimizer to use. Required if EnableCompaction is - * true. Can be the same role as OptimizerRoleArn if both optimizers are - * enabled. See [AWS + * true. Can be the same role as OptimizerRoleArn or OrphanFileDeletionRoleArn + * if multiple optimizers are enabled. See [AWS * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). * @param {String} [options.CompactionStrategy='binpack'] - The compaction * strategy: binpack, sort, or z-order. See [AWS @@ -46,6 +46,20 @@ const GlueTable = require('./glue-table'); * before compaction triggers. * @param {Number} [options.DeleteFileThreshold=1] - Minimum deletes in a file * to make it eligible for compaction. + * @param {Boolean} [options.EnableOrphanFileDeletion=false] - Whether to + * enable the orphan file deletion optimizer for this Iceberg table. + * @param {String} [options.OrphanFileDeletionRoleArn=undefined] - The ARN of + * the IAM role for the orphan file deletion optimizer to use. Required if + * EnableOrphanFileDeletion is true. Can be the same role as OptimizerRoleArn + * or CompactionRoleArn if multiple optimizers are enabled. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). + * @param {Number} [options.OrphanFileRetentionPeriodInDays=3] - The number of + * days to retain orphan files before deleting them. See [AWS + * documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). + * @param {String} [options.OrphanFileDeletionLocation=undefined] - The S3 + * location to scan for orphan files. Defaults to the table location if not + * specified. See [AWS + * documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). */ class GlueIcebergTable extends GlueTable { constructor(options) { @@ -62,7 +76,11 @@ class GlueIcebergTable extends GlueTable { CompactionRoleArn, CompactionStrategy = 'binpack', MinInputFiles = 100, - DeleteFileThreshold = 1 + DeleteFileThreshold = 1, + EnableOrphanFileDeletion = false, + OrphanFileDeletionRoleArn, + OrphanFileRetentionPeriodInDays = 3, + OrphanFileDeletionLocation } = options; const required = [Location]; @@ -75,6 +93,9 @@ class GlueIcebergTable extends GlueTable { if (EnableCompaction && !CompactionRoleArn) throw new Error('You must provide a CompactionRoleArn when EnableCompaction is true'); + if (EnableOrphanFileDeletion && !OrphanFileDeletionRoleArn) + throw new Error('You must provide an OrphanFileDeletionRoleArn when EnableOrphanFileDeletion is true'); + const validStrategies = ['binpack', 'sort', 'z-order']; if (!validStrategies.includes(CompactionStrategy)) throw new Error('CompactionStrategy must be one of: binpack, sort, z-order'); @@ -158,6 +179,42 @@ class GlueIcebergTable extends GlueTable { this.Resources[compactionLogicalName].Condition = options.Condition; } } + + // Optionally add TableOptimizer for orphan file deletion + if (EnableOrphanFileDeletion) { + const orphanLogicalName = `${logicalName}OrphanFileDeletionOptimizer`; + const icebergConfiguration = { + OrphanFileRetentionPeriodInDays + }; + + // Only add Location if specified, otherwise it defaults to table location + if (OrphanFileDeletionLocation) { + icebergConfiguration.Location = OrphanFileDeletionLocation; + } + + this.Resources[orphanLogicalName] = { + Type: 'AWS::Glue::TableOptimizer', + DependsOn: logicalName, + Properties: { + CatalogId: options.CatalogId || { Ref: 'AWS::AccountId' }, + DatabaseName: options.DatabaseName, + TableName: options.Name, + Type: 'orphan_file_deletion', + TableOptimizerConfiguration: { + RoleArn: OrphanFileDeletionRoleArn, + Enabled: true, + OrphanFileDeletionConfiguration: { + IcebergConfiguration: icebergConfiguration + } + } + } + }; + + // Apply Condition to orphan file deletion optimizer if specified on the table + if (options.Condition) { + this.Resources[orphanLogicalName].Condition = options.Condition; + } + } } } diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json b/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json new file mode 100644 index 0000000..fc26a6a --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json @@ -0,0 +1,117 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableRetentionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/SharedRole", + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 5, + "NumberOfSnapshotsToRetain": 1, + "CleanExpiredFiles": true + } + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/SharedRole", + "Enabled": true, + "CompactionConfiguration": { + "IcebergConfiguration": { + "Strategy": "binpack", + "MinInputFiles": 100, + "DeleteFileThreshold": 1 + } + } + } + } + }, + "MyTableOrphanFileDeletionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "orphan_file_deletion", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/SharedRole", + "Enabled": true, + "OrphanFileDeletionConfiguration": { + "IcebergConfiguration": { + "OrphanFileRetentionPeriodInDays": 3 + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json new file mode 100644 index 0000000..dd776ab --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json @@ -0,0 +1,83 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "OrphanFileDeletionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": {} + } + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableOrphanFileDeletionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "orphan_file_deletion", + "TableOptimizerConfiguration": { + "RoleArn": { + "Fn::GetAtt": [ + "OrphanFileDeletionRole", + "Arn" + ] + }, + "Enabled": true, + "OrphanFileDeletionConfiguration": { + "IcebergConfiguration": { + "OrphanFileRetentionPeriodInDays": 7, + "Location": "s3://fake/location/subdir" + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json new file mode 100644 index 0000000..e710c29 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json @@ -0,0 +1,71 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableOrphanFileDeletionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "orphan_file_deletion", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/OrphanFileDeletionRole", + "Enabled": true, + "OrphanFileDeletionConfiguration": { + "IcebergConfiguration": { + "OrphanFileRetentionPeriodInDays": 3 + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/shortcuts.test.js b/test/shortcuts.test.js index 857f683..02153dc 100644 --- a/test/shortcuts.test.js +++ b/test/shortcuts.test.js @@ -1646,6 +1646,90 @@ test('[shortcuts] glue iceberg table', (assert) => { 'expected resources generated with both retention and compaction optimizers' ); + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOrphanFileDeletion: true + }), + /You must provide an OrphanFileDeletionRoleArn when EnableOrphanFileDeletion is true/, + 'throws when EnableOrphanFileDeletion is true but OrphanFileDeletionRoleArn is missing' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOrphanFileDeletion: true, + OrphanFileDeletionRoleArn: 'arn:aws:iam::123456789012:role/OrphanFileDeletionRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-orphan-deletion-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-orphan-deletion-defaults'), + 'expected resources generated with orphan file deletion using default settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOrphanFileDeletion: true, + OrphanFileDeletionRoleArn: cf.getAtt('OrphanFileDeletionRole', 'Arn'), + OrphanFileRetentionPeriodInDays: 7, + OrphanFileDeletionLocation: 's3://fake/location/subdir' + }); + + template = cf.merge( + { Resources: { OrphanFileDeletionRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-with-orphan-deletion-custom', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-orphan-deletion-custom'), + 'expected resources generated with orphan file deletion using custom settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: 'arn:aws:iam::123456789012:role/SharedRole', + EnableCompaction: true, + CompactionRoleArn: 'arn:aws:iam::123456789012:role/SharedRole', + EnableOrphanFileDeletion: true, + OrphanFileDeletionRoleArn: 'arn:aws:iam::123456789012:role/SharedRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-all-optimizers', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-all-optimizers'), + 'expected resources generated with all three optimizers using same role' + ); + assert.end(); }); From 6b02cec92e6624ca0bcaf6f83484a45d93504572 Mon Sep 17 00:00:00 2001 From: jeffhiltz Date: Fri, 19 Dec 2025 17:05:14 -0500 Subject: [PATCH 4/6] remove the compaction config which is in the api but not CFN --- lib/shortcuts/api.md | 5 +-- lib/shortcuts/glue-iceberg-table.js | 32 ++++++------------- ...lue-iceberg-table-with-all-optimizers.json | 9 +----- ...ue-iceberg-table-with-both-optimizers.json | 9 +----- ...-iceberg-table-with-compaction-custom.json | 9 +----- ...ceberg-table-with-compaction-defaults.json | 9 +----- test/shortcuts.test.js | 25 ++------------- 7 files changed, 16 insertions(+), 82 deletions(-) diff --git a/lib/shortcuts/api.md b/lib/shortcuts/api.md index db38698..c1e35a7 100644 --- a/lib/shortcuts/api.md +++ b/lib/shortcuts/api.md @@ -226,11 +226,8 @@ Create a Glue table backed by Apache Iceberg format on S3. | [options.SnapshotRetentionPeriodInDays] | Number | 5 | The number of days to retain snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | | [options.NumberOfSnapshotsToRetain] | Number | 1 | The minimum number of snapshots to retain. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | | [options.CleanExpiredFiles] | Boolean | true | Whether to delete expired data files after expiring snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | -| [options.EnableCompaction] | Boolean | false | Whether to enable the compaction optimizer for this Iceberg table. | +| [options.EnableCompaction] | Boolean | false | Whether to enable the compaction optimizer for this Iceberg table. Note: CloudFormation does not support configuring compaction strategy or thresholds; the optimizer will use AWS defaults (binpack strategy). Configuration must be done via AWS CLI/API. See [GitHub issue](https://github.com/aws-cloudformation/cloudformation-coverage-roadmap/issues/2257). | | [options.CompactionRoleArn] | String | | The ARN of the IAM role for the compaction optimizer to use. Required if EnableCompaction is true. Can be the same role as OptimizerRoleArn or OrphanFileDeletionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | -| [options.CompactionStrategy] | String | 'binpack' | The compaction strategy: binpack, sort, or z-order. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-compaction.html). | -| [options.MinInputFiles] | Number | 100 | Minimum number of data files before compaction triggers. | -| [options.DeleteFileThreshold] | Number | 1 | Minimum deletes in a file to make it eligible for compaction. | | [options.EnableOrphanFileDeletion] | Boolean | false | Whether to enable the orphan file deletion optimizer for this Iceberg table. | | [options.OrphanFileDeletionRoleArn] | String | | The ARN of the IAM role for the orphan file deletion optimizer to use. Required if EnableOrphanFileDeletion is true. Can be the same role as OptimizerRoleArn or CompactionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | | [options.OrphanFileRetentionPeriodInDays] | Number | 3 | The number of days to retain orphan files before deleting them. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). | diff --git a/lib/shortcuts/glue-iceberg-table.js b/lib/shortcuts/glue-iceberg-table.js index d9ccf05..8b3c7bb 100644 --- a/lib/shortcuts/glue-iceberg-table.js +++ b/lib/shortcuts/glue-iceberg-table.js @@ -33,19 +33,15 @@ const GlueTable = require('./glue-table'); * expired data files after expiring snapshots. See [AWS * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). * @param {Boolean} [options.EnableCompaction=false] - Whether to enable the - * compaction optimizer for this Iceberg table. + * compaction optimizer for this Iceberg table. Note: CloudFormation does not + * support configuring compaction strategy or thresholds; the optimizer will use + * AWS defaults (binpack strategy). Configuration must be done via AWS CLI/API. + * See [GitHub issue](https://github.com/aws-cloudformation/cloudformation-coverage-roadmap/issues/2257). * @param {String} [options.CompactionRoleArn=undefined] - The ARN of the IAM * role for the compaction optimizer to use. Required if EnableCompaction is * true. Can be the same role as OptimizerRoleArn or OrphanFileDeletionRoleArn * if multiple optimizers are enabled. See [AWS * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). - * @param {String} [options.CompactionStrategy='binpack'] - The compaction - * strategy: binpack, sort, or z-order. See [AWS - * documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-compaction.html). - * @param {Number} [options.MinInputFiles=100] - Minimum number of data files - * before compaction triggers. - * @param {Number} [options.DeleteFileThreshold=1] - Minimum deletes in a file - * to make it eligible for compaction. * @param {Boolean} [options.EnableOrphanFileDeletion=false] - Whether to * enable the orphan file deletion optimizer for this Iceberg table. * @param {String} [options.OrphanFileDeletionRoleArn=undefined] - The ARN of @@ -74,9 +70,6 @@ class GlueIcebergTable extends GlueTable { CleanExpiredFiles = true, EnableCompaction = false, CompactionRoleArn, - CompactionStrategy = 'binpack', - MinInputFiles = 100, - DeleteFileThreshold = 1, EnableOrphanFileDeletion = false, OrphanFileDeletionRoleArn, OrphanFileRetentionPeriodInDays = 3, @@ -96,10 +89,6 @@ class GlueIcebergTable extends GlueTable { if (EnableOrphanFileDeletion && !OrphanFileDeletionRoleArn) throw new Error('You must provide an OrphanFileDeletionRoleArn when EnableOrphanFileDeletion is true'); - const validStrategies = ['binpack', 'sort', 'z-order']; - if (!validStrategies.includes(CompactionStrategy)) - throw new Error('CompactionStrategy must be one of: binpack, sort, z-order'); - super( Object.assign( { @@ -150,6 +139,10 @@ class GlueIcebergTable extends GlueTable { } // Optionally add TableOptimizer for compaction + // NOTE: CloudFormation does not support CompactionConfiguration properties + // (strategy, minInputFiles, deleteFileThreshold). These must be configured + // via AWS CLI/API after stack creation, or will use AWS defaults. + // See: https://github.com/aws-cloudformation/cloudformation-coverage-roadmap/issues/2257 if (EnableCompaction) { const compactionLogicalName = `${logicalName}CompactionOptimizer`; this.Resources[compactionLogicalName] = { @@ -162,14 +155,7 @@ class GlueIcebergTable extends GlueTable { Type: 'compaction', TableOptimizerConfiguration: { RoleArn: CompactionRoleArn, - Enabled: true, - CompactionConfiguration: { - IcebergConfiguration: { - Strategy: CompactionStrategy, - MinInputFiles, - DeleteFileThreshold - } - } + Enabled: true } } }; diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json b/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json index fc26a6a..f2d181a 100644 --- a/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json @@ -80,14 +80,7 @@ "Type": "compaction", "TableOptimizerConfiguration": { "RoleArn": "arn:aws:iam::123456789012:role/SharedRole", - "Enabled": true, - "CompactionConfiguration": { - "IcebergConfiguration": { - "Strategy": "binpack", - "MinInputFiles": 100, - "DeleteFileThreshold": 1 - } - } + "Enabled": true } } }, diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json b/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json index 5874a2c..a9fe1d6 100644 --- a/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json @@ -80,14 +80,7 @@ "Type": "compaction", "TableOptimizerConfiguration": { "RoleArn": "arn:aws:iam::123456789012:role/CompactionRole", - "Enabled": true, - "CompactionConfiguration": { - "IcebergConfiguration": { - "Strategy": "sort", - "MinInputFiles": 100, - "DeleteFileThreshold": 1 - } - } + "Enabled": true } } } diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json index abe9583..c4ffe91 100644 --- a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json @@ -68,14 +68,7 @@ "Arn" ] }, - "Enabled": true, - "CompactionConfiguration": { - "IcebergConfiguration": { - "Strategy": "z-order", - "MinInputFiles": 50, - "DeleteFileThreshold": 10 - } - } + "Enabled": true } } } diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json index 79476b1..9948d48 100644 --- a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json @@ -57,14 +57,7 @@ "Type": "compaction", "TableOptimizerConfiguration": { "RoleArn": "arn:aws:iam::123456789012:role/CompactionRole", - "Enabled": true, - "CompactionConfiguration": { - "IcebergConfiguration": { - "Strategy": "binpack", - "MinInputFiles": 100, - "DeleteFileThreshold": 1 - } - } + "Enabled": true } } } diff --git a/test/shortcuts.test.js b/test/shortcuts.test.js index 02153dc..6aa465b 100644 --- a/test/shortcuts.test.js +++ b/test/shortcuts.test.js @@ -1560,23 +1560,6 @@ test('[shortcuts] glue iceberg table', (assert) => { 'throws when EnableCompaction is true but CompactionRoleArn is missing' ); - assert.throws( - () => new cf.shortcuts.GlueIcebergTable({ - LogicalName: 'MyTable', - DatabaseName: 'my_database', - Name: 'my_table', - Columns: [ - { Name: 'column', Type: 'string' } - ], - Location: 's3://fake/location', - EnableCompaction: true, - CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole', - CompactionStrategy: 'invalid' - }), - /CompactionStrategy must be one of: binpack, sort, z-order/, - 'throws when CompactionStrategy is invalid' - ); - db = new cf.shortcuts.GlueIcebergTable({ LogicalName: 'MyTable', DatabaseName: 'my_database', @@ -1606,10 +1589,7 @@ test('[shortcuts] glue iceberg table', (assert) => { ], Location: 's3://fake/location', EnableCompaction: true, - CompactionRoleArn: cf.getAtt('CompactionRole', 'Arn'), - CompactionStrategy: 'z-order', - MinInputFiles: 50, - DeleteFileThreshold: 10 + CompactionRoleArn: cf.getAtt('CompactionRole', 'Arn') }); template = cf.merge( @@ -1634,8 +1614,7 @@ test('[shortcuts] glue iceberg table', (assert) => { EnableOptimizer: true, OptimizerRoleArn: 'arn:aws:iam::123456789012:role/RetentionRole', EnableCompaction: true, - CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole', - CompactionStrategy: 'sort' + CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole' }); template = cf.merge(db); From 87e833368f2584518151cb078928002af0033097 Mon Sep 17 00:00:00 2001 From: jeffhiltz Date: Fri, 19 Dec 2025 17:11:47 -0500 Subject: [PATCH 5/6] update cfn-lint to get newer property types --- requirements.dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index f76f9b7..7582496 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,2 +1,2 @@ aws-sam-cli==1.142.1 -cfn-lint==1.36.1 +cfn-lint==1.43.1 From 7c01d4888fddb7390e1ff4179362313413575ae8 Mon Sep 17 00:00:00 2001 From: jeffhiltz Date: Fri, 19 Dec 2025 17:32:39 -0500 Subject: [PATCH 6/6] update sam-cli in order to update cfn-lint --- requirements.dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 7582496..3e6c219 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,2 +1,2 @@ -aws-sam-cli==1.142.1 -cfn-lint==1.43.1 +aws-sam-cli==1.149.0 +cfn-lint==1.41.0