diff --git a/lib/shortcuts/api.md b/lib/shortcuts/api.md index 8f93a2b..c1e35a7 100644 --- a/lib/shortcuts/api.md +++ b/lib/shortcuts/api.md @@ -12,6 +12,9 @@ a Lambda permission.

GlueDatabase

Create a Glue Database.

+
GlueIcebergTable
+

Create a Glue table backed by Apache Iceberg format on S3.

+
GlueJsonTable

Create a Glue Table backed by line-delimited JSON files on S3.

@@ -202,6 +205,34 @@ const db = new cf.shortcuts.GlueDatabase({ module.exports = cf.merge(myTemplate, db); ``` + + +## GlueIcebergTable +Create a Glue table backed by Apache Iceberg format on S3. + +**Kind**: global class + + +### new GlueIcebergTable(options) + +| Param | Type | Default | Description | +| --- | --- | --- | --- | +| options | Object | | Accepts the same options as cloudfriend's [`GlueTable`](https://github.com/mapbox/cloudfriend/blob/master/lib/shortcuts/glue-table.js), though the following additional attributes are either required or hard-wired: | +| options.Location | String | | The physical location of the table. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-storagedescriptor.html#cfn-glue-table-storagedescriptor-location). | +| [options.TableType] | String | 'EXTERNAL_TABLE' | Hard-wired by this shortcut. | +| [options.IcebergVersion] | String | '2' | The table version for the Iceberg table. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-iceberginput.html). | +| [options.EnableOptimizer] | Boolean | false | Whether to enable the snapshot retention optimizer for this Iceberg table. | +| [options.OptimizerRoleArn] | String | | The ARN of the IAM role for the retention optimizer to use. Required if EnableOptimizer is true. Can be the same role as CompactionRoleArn or OrphanFileDeletionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.SnapshotRetentionPeriodInDays] | Number | 5 | The number of days to retain snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | +| [options.NumberOfSnapshotsToRetain] | Number | 1 | The minimum number of snapshots to retain. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | +| [options.CleanExpiredFiles] | Boolean | true | Whether to delete expired data files after expiring snapshots. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). | +| [options.EnableCompaction] | Boolean | false | Whether to enable the compaction optimizer for this Iceberg table. Note: CloudFormation does not support configuring compaction strategy or thresholds; the optimizer will use AWS defaults (binpack strategy). Configuration must be done via AWS CLI/API. See [GitHub issue](https://github.com/aws-cloudformation/cloudformation-coverage-roadmap/issues/2257). | +| [options.CompactionRoleArn] | String | | The ARN of the IAM role for the compaction optimizer to use. Required if EnableCompaction is true. Can be the same role as OptimizerRoleArn or OrphanFileDeletionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.EnableOrphanFileDeletion] | Boolean | false | Whether to enable the orphan file deletion optimizer for this Iceberg table. | +| [options.OrphanFileDeletionRoleArn] | String | | The ARN of the IAM role for the orphan file deletion optimizer to use. Required if EnableOrphanFileDeletion is true. Can be the same role as OptimizerRoleArn or CompactionRoleArn if multiple optimizers are enabled. See [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). | +| [options.OrphanFileRetentionPeriodInDays] | Number | 3 | The number of days to retain orphan files before deleting them. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). | +| [options.OrphanFileDeletionLocation] | String | | The S3 location to scan for orphan files. Defaults to the table location if not specified. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). | + ## GlueJsonTable diff --git a/lib/shortcuts/glue-iceberg-table.js b/lib/shortcuts/glue-iceberg-table.js new file mode 100644 index 0000000..8b3c7bb --- /dev/null +++ b/lib/shortcuts/glue-iceberg-table.js @@ -0,0 +1,207 @@ +'use strict'; + +const GlueTable = require('./glue-table'); + +/** + * Create a Glue table backed by Apache Iceberg format on S3. + * + * @param {Object} options - Accepts the same options as cloudfriend's + * [`GlueTable`](https://github.com/mapbox/cloudfriend/blob/master/lib/shortcuts/glue-table.js), + * though the following additional attributes are either required or hard-wired: + * @param {String} options.Location - The physical location of the table. See + * [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-storagedescriptor.html#cfn-glue-table-storagedescriptor-location). + * @param {String} [options.TableType='EXTERNAL_TABLE'] - Hard-wired by this + * shortcut. + * @param {String} [options.IcebergVersion='2'] - The table version for the + * Iceberg table. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-iceberginput.html). + * @param {Boolean} [options.EnableOptimizer=false] - Whether to enable the + * snapshot retention optimizer for this Iceberg table. + * @param {String} [options.OptimizerRoleArn=undefined] - The ARN of the IAM + * role for the retention optimizer to use. Required if EnableOptimizer is + * true. Can be the same role as CompactionRoleArn or OrphanFileDeletionRoleArn + * if multiple optimizers are enabled. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). + * @param {Number} [options.SnapshotRetentionPeriodInDays=5] - The number of + * days to retain snapshots. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). + * @param {Number} [options.NumberOfSnapshotsToRetain=1] - The minimum number + * of snapshots to retain. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). + * @param {Boolean} [options.CleanExpiredFiles=true] - Whether to delete + * expired data files after expiring snapshots. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-glue-tableoptimizer-icebergretentionconfiguration.html). + * @param {Boolean} [options.EnableCompaction=false] - Whether to enable the + * compaction optimizer for this Iceberg table. Note: CloudFormation does not + * support configuring compaction strategy or thresholds; the optimizer will use + * AWS defaults (binpack strategy). Configuration must be done via AWS CLI/API. + * See [GitHub issue](https://github.com/aws-cloudformation/cloudformation-coverage-roadmap/issues/2257). + * @param {String} [options.CompactionRoleArn=undefined] - The ARN of the IAM + * role for the compaction optimizer to use. Required if EnableCompaction is + * true. Can be the same role as OptimizerRoleArn or OrphanFileDeletionRoleArn + * if multiple optimizers are enabled. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). + * @param {Boolean} [options.EnableOrphanFileDeletion=false] - Whether to + * enable the orphan file deletion optimizer for this Iceberg table. + * @param {String} [options.OrphanFileDeletionRoleArn=undefined] - The ARN of + * the IAM role for the orphan file deletion optimizer to use. Required if + * EnableOrphanFileDeletion is true. Can be the same role as OptimizerRoleArn + * or CompactionRoleArn if multiple optimizers are enabled. See [AWS + * documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-tableoptimizer-tableoptimizerconfiguration.html). + * @param {Number} [options.OrphanFileRetentionPeriodInDays=3] - The number of + * days to retain orphan files before deleting them. See [AWS + * documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). + * @param {String} [options.OrphanFileDeletionLocation=undefined] - The S3 + * location to scan for orphan files. Defaults to the table location if not + * specified. See [AWS + * documentation](https://docs.aws.amazon.com/glue/latest/dg/enable-orphan-file-deletion.html). + */ +class GlueIcebergTable extends GlueTable { + constructor(options) { + if (!options) throw new Error('Options required'); + const { + Location, + IcebergVersion = '2', + EnableOptimizer = false, + OptimizerRoleArn, + SnapshotRetentionPeriodInDays = 5, + NumberOfSnapshotsToRetain = 1, + CleanExpiredFiles = true, + EnableCompaction = false, + CompactionRoleArn, + EnableOrphanFileDeletion = false, + OrphanFileDeletionRoleArn, + OrphanFileRetentionPeriodInDays = 3, + OrphanFileDeletionLocation + } = options; + + const required = [Location]; + if (required.some((variable) => !variable)) + throw new Error('You must provide a Location'); + + if (EnableOptimizer && !OptimizerRoleArn) + throw new Error('You must provide an OptimizerRoleArn when EnableOptimizer is true'); + + if (EnableCompaction && !CompactionRoleArn) + throw new Error('You must provide a CompactionRoleArn when EnableCompaction is true'); + + if (EnableOrphanFileDeletion && !OrphanFileDeletionRoleArn) + throw new Error('You must provide an OrphanFileDeletionRoleArn when EnableOrphanFileDeletion is true'); + + super( + Object.assign( + { + TableType: 'EXTERNAL_TABLE', + Parameters: { EXTERNAL: 'TRUE' } + }, + options + ) + ); + + const logicalName = options.LogicalName; + this.Resources[logicalName].Properties.OpenTableFormatInput = { + IcebergInput: { + MetadataOperation: 'CREATE', + Version: IcebergVersion + } + }; + + // Optionally add TableOptimizer for configuring snapshot retention + if (EnableOptimizer) { + const optimizerLogicalName = `${logicalName}RetentionOptimizer`; + this.Resources[optimizerLogicalName] = { + Type: 'AWS::Glue::TableOptimizer', + DependsOn: logicalName, + Properties: { + CatalogId: options.CatalogId || { Ref: 'AWS::AccountId' }, + DatabaseName: options.DatabaseName, + TableName: options.Name, + Type: 'retention', + TableOptimizerConfiguration: { + RoleArn: OptimizerRoleArn, + Enabled: true, + RetentionConfiguration: { + IcebergConfiguration: { + SnapshotRetentionPeriodInDays, + NumberOfSnapshotsToRetain, + CleanExpiredFiles + } + } + } + } + }; + + // Apply Condition to optimizer if specified on the table + if (options.Condition) { + this.Resources[optimizerLogicalName].Condition = options.Condition; + } + } + + // Optionally add TableOptimizer for compaction + // NOTE: CloudFormation does not support CompactionConfiguration properties + // (strategy, minInputFiles, deleteFileThreshold). These must be configured + // via AWS CLI/API after stack creation, or will use AWS defaults. + // See: https://github.com/aws-cloudformation/cloudformation-coverage-roadmap/issues/2257 + if (EnableCompaction) { + const compactionLogicalName = `${logicalName}CompactionOptimizer`; + this.Resources[compactionLogicalName] = { + Type: 'AWS::Glue::TableOptimizer', + DependsOn: logicalName, + Properties: { + CatalogId: options.CatalogId || { Ref: 'AWS::AccountId' }, + DatabaseName: options.DatabaseName, + TableName: options.Name, + Type: 'compaction', + TableOptimizerConfiguration: { + RoleArn: CompactionRoleArn, + Enabled: true + } + } + }; + + // Apply Condition to compaction optimizer if specified on the table + if (options.Condition) { + this.Resources[compactionLogicalName].Condition = options.Condition; + } + } + + // Optionally add TableOptimizer for orphan file deletion + if (EnableOrphanFileDeletion) { + const orphanLogicalName = `${logicalName}OrphanFileDeletionOptimizer`; + const icebergConfiguration = { + OrphanFileRetentionPeriodInDays + }; + + // Only add Location if specified, otherwise it defaults to table location + if (OrphanFileDeletionLocation) { + icebergConfiguration.Location = OrphanFileDeletionLocation; + } + + this.Resources[orphanLogicalName] = { + Type: 'AWS::Glue::TableOptimizer', + DependsOn: logicalName, + Properties: { + CatalogId: options.CatalogId || { Ref: 'AWS::AccountId' }, + DatabaseName: options.DatabaseName, + TableName: options.Name, + Type: 'orphan_file_deletion', + TableOptimizerConfiguration: { + RoleArn: OrphanFileDeletionRoleArn, + Enabled: true, + OrphanFileDeletionConfiguration: { + IcebergConfiguration: icebergConfiguration + } + } + } + }; + + // Apply Condition to orphan file deletion optimizer if specified on the table + if (options.Condition) { + this.Resources[orphanLogicalName].Condition = options.Condition; + } + } + } +} + +module.exports = GlueIcebergTable; diff --git a/lib/shortcuts/index.js b/lib/shortcuts/index.js index 54e40fe..63fa0f9 100644 --- a/lib/shortcuts/index.js +++ b/lib/shortcuts/index.js @@ -16,6 +16,7 @@ module.exports = { GlueJsonTable: require('./glue-json-table'), GlueOrcTable: require('./glue-orc-table'), GlueParquetTable: require('./glue-parquet-table'), + GlueIcebergTable: require('./glue-iceberg-table'), GluePrestoView: require('./glue-presto-view'), GlueSparkView: require('./glue-spark-view'), hookshot: require('./hookshot'), diff --git a/requirements.dev.txt b/requirements.dev.txt index f76f9b7..3e6c219 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,2 +1,2 @@ -aws-sam-cli==1.142.1 -cfn-lint==1.36.1 +aws-sam-cli==1.149.0 +cfn-lint==1.41.0 diff --git a/test/fixtures/shortcuts/glue-iceberg-table-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-defaults.json new file mode 100644 index 0000000..2d00ccf --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-defaults.json @@ -0,0 +1,50 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json new file mode 100644 index 0000000..3c16713 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json @@ -0,0 +1,60 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": { + "Always": { + "Fn::Equals": [ + "1", + "1" + ] + } + }, + "Resources": { + "AnotherThing": { + "Type": "AWS::SNS::Topic" + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Condition": "Always", + "DependsOn": "AnotherThing", + "Properties": { + "CatalogId": "1234", + "DatabaseName": "my_database", + "TableInput": { + "Description": "my_table description", + "Name": "my_table", + "Owner": "Team", + "Parameters": { + "table": "params" + }, + "PartitionKeys": [], + "Retention": 12, + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json b/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json new file mode 100644 index 0000000..f2d181a --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json @@ -0,0 +1,110 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableRetentionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/SharedRole", + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 5, + "NumberOfSnapshotsToRetain": 1, + "CleanExpiredFiles": true + } + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/SharedRole", + "Enabled": true + } + } + }, + "MyTableOrphanFileDeletionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "orphan_file_deletion", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/SharedRole", + "Enabled": true, + "OrphanFileDeletionConfiguration": { + "IcebergConfiguration": { + "OrphanFileRetentionPeriodInDays": 3 + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json b/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json new file mode 100644 index 0000000..a9fe1d6 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-both-optimizers.json @@ -0,0 +1,89 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableRetentionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/RetentionRole", + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 5, + "NumberOfSnapshotsToRetain": 1, + "CleanExpiredFiles": true + } + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/CompactionRole", + "Enabled": true + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json new file mode 100644 index 0000000..c4ffe91 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-custom.json @@ -0,0 +1,77 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "CompactionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": {} + } + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": { + "Fn::GetAtt": [ + "CompactionRole", + "Arn" + ] + }, + "Enabled": true + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json new file mode 100644 index 0000000..9948d48 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-compaction-defaults.json @@ -0,0 +1,66 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableCompactionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "compaction", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/CompactionRole", + "Enabled": true + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json new file mode 100644 index 0000000..cc3ed38 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json @@ -0,0 +1,93 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": { + "Always": { + "Fn::Equals": [ + "1", + "1" + ] + } + }, + "Resources": { + "OptimizerRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": {} + } + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Condition": "Always", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableRetentionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": { + "Fn::GetAtt": [ + "OptimizerRole", + "Arn" + ] + }, + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 7, + "NumberOfSnapshotsToRetain": 3, + "CleanExpiredFiles": false + } + } + } + }, + "Condition": "Always" + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json new file mode 100644 index 0000000..742cb3f --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json @@ -0,0 +1,73 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableRetentionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "retention", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/OptimizerRole", + "Enabled": true, + "RetentionConfiguration": { + "IcebergConfiguration": { + "SnapshotRetentionPeriodInDays": 5, + "NumberOfSnapshotsToRetain": 1, + "CleanExpiredFiles": true + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json new file mode 100644 index 0000000..dd776ab --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json @@ -0,0 +1,83 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "OrphanFileDeletionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": {} + } + }, + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableOrphanFileDeletionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "orphan_file_deletion", + "TableOptimizerConfiguration": { + "RoleArn": { + "Fn::GetAtt": [ + "OrphanFileDeletionRole", + "Arn" + ] + }, + "Enabled": true, + "OrphanFileDeletionConfiguration": { + "IcebergConfiguration": { + "OrphanFileRetentionPeriodInDays": 7, + "Location": "s3://fake/location/subdir" + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json new file mode 100644 index 0000000..e710c29 --- /dev/null +++ b/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json @@ -0,0 +1,71 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": {}, + "Parameters": {}, + "Rules": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": { + "MyTable": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableInput": { + "Description": { + "Fn::Sub": "Created by the ${AWS::StackName} CloudFormation stack" + }, + "Name": "my_table", + "Parameters": { + "EXTERNAL": "TRUE" + }, + "PartitionKeys": [], + "TableType": "EXTERNAL_TABLE", + "StorageDescriptor": { + "Columns": [ + { + "Name": "column", + "Type": "string" + } + ], + "Compressed": false, + "Location": "s3://fake/location", + "NumberOfBuckets": 0, + "SerdeInfo": {}, + "StoredAsSubDirectories": true + } + }, + "OpenTableFormatInput": { + "IcebergInput": { + "MetadataOperation": "CREATE", + "Version": "2" + } + } + } + }, + "MyTableOrphanFileDeletionOptimizer": { + "Type": "AWS::Glue::TableOptimizer", + "DependsOn": "MyTable", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": "my_database", + "TableName": "my_table", + "Type": "orphan_file_deletion", + "TableOptimizerConfiguration": { + "RoleArn": "arn:aws:iam::123456789012:role/OrphanFileDeletionRole", + "Enabled": true, + "OrphanFileDeletionConfiguration": { + "IcebergConfiguration": { + "OrphanFileRetentionPeriodInDays": 3 + } + } + } + } + } + }, + "Outputs": {} +} \ No newline at end of file diff --git a/test/shortcuts.test.js b/test/shortcuts.test.js index 85f222d..6aa465b 100644 --- a/test/shortcuts.test.js +++ b/test/shortcuts.test.js @@ -1422,6 +1422,296 @@ test('[shortcuts] glue parquet table', (assert) => { assert.end(); }); +test('[shortcuts] glue iceberg table', (assert) => { + assert.throws( + () => new cf.shortcuts.GlueIcebergTable(), + 'Options required', + 'throws without options' + ); + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({}), + /You must provide a Location/, + 'throws without required parameters' + ); + + let db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location' + }); + + let template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-defaults'), + 'expected resources generated with defaults' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + CatalogId: '1234', + Owner: 'Team', + Parameters: { table: 'params' }, + Description: 'my_table description', + Retention: 12, + Location: 's3://fake/location', + IcebergVersion: '2', + Condition: 'Always', + DependsOn: 'AnotherThing' + }); + + template = cf.merge( + { Conditions: { Always: cf.equals('1', '1') } }, + { Resources: { AnotherThing: { Type: 'AWS::SNS::Topic' } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-no-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-no-defaults'), + 'expected resources generated without defaults' + ); + + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true + }), + /You must provide an OptimizerRoleArn when EnableOptimizer is true/, + 'throws when EnableOptimizer is true but OptimizerRoleArn is missing' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: 'arn:aws:iam::123456789012:role/OptimizerRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-optimizer-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-optimizer-defaults'), + 'expected resources generated with optimizer using default retention settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: cf.getAtt('OptimizerRole', 'Arn'), + SnapshotRetentionPeriodInDays: 7, + NumberOfSnapshotsToRetain: 3, + CleanExpiredFiles: false, + Condition: 'Always' + }); + + template = cf.merge( + { Conditions: { Always: cf.equals('1', '1') } }, + { Resources: { OptimizerRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-with-optimizer-custom', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-optimizer-custom'), + 'expected resources generated with optimizer using custom retention settings' + ); + + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableCompaction: true + }), + /You must provide a CompactionRoleArn when EnableCompaction is true/, + 'throws when EnableCompaction is true but CompactionRoleArn is missing' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableCompaction: true, + CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-compaction-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-compaction-defaults'), + 'expected resources generated with compaction using default settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableCompaction: true, + CompactionRoleArn: cf.getAtt('CompactionRole', 'Arn') + }); + + template = cf.merge( + { Resources: { CompactionRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-with-compaction-custom', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-compaction-custom'), + 'expected resources generated with compaction using custom settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: 'arn:aws:iam::123456789012:role/RetentionRole', + EnableCompaction: true, + CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-both-optimizers', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-both-optimizers'), + 'expected resources generated with both retention and compaction optimizers' + ); + + assert.throws( + () => new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOrphanFileDeletion: true + }), + /You must provide an OrphanFileDeletionRoleArn when EnableOrphanFileDeletion is true/, + 'throws when EnableOrphanFileDeletion is true but OrphanFileDeletionRoleArn is missing' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOrphanFileDeletion: true, + OrphanFileDeletionRoleArn: 'arn:aws:iam::123456789012:role/OrphanFileDeletionRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-orphan-deletion-defaults', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-orphan-deletion-defaults'), + 'expected resources generated with orphan file deletion using default settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOrphanFileDeletion: true, + OrphanFileDeletionRoleArn: cf.getAtt('OrphanFileDeletionRole', 'Arn'), + OrphanFileRetentionPeriodInDays: 7, + OrphanFileDeletionLocation: 's3://fake/location/subdir' + }); + + template = cf.merge( + { Resources: { OrphanFileDeletionRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } }, + db + ); + if (update) fixtures.update('glue-iceberg-table-with-orphan-deletion-custom', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-orphan-deletion-custom'), + 'expected resources generated with orphan file deletion using custom settings' + ); + + db = new cf.shortcuts.GlueIcebergTable({ + LogicalName: 'MyTable', + DatabaseName: 'my_database', + Name: 'my_table', + Columns: [ + { Name: 'column', Type: 'string' } + ], + Location: 's3://fake/location', + EnableOptimizer: true, + OptimizerRoleArn: 'arn:aws:iam::123456789012:role/SharedRole', + EnableCompaction: true, + CompactionRoleArn: 'arn:aws:iam::123456789012:role/SharedRole', + EnableOrphanFileDeletion: true, + OrphanFileDeletionRoleArn: 'arn:aws:iam::123456789012:role/SharedRole' + }); + + template = cf.merge(db); + if (update) fixtures.update('glue-iceberg-table-with-all-optimizers', template); + assert.deepEqual( + noUndefined(template), + fixtures.get('glue-iceberg-table-with-all-optimizers'), + 'expected resources generated with all three optimizers using same role' + ); + + assert.end(); +}); + test('[shortcuts] glue view', (assert) => { assert.throws( () => new cf.shortcuts.GluePrestoView(),