Merge pull request #1383 from hkad98/jkd/geo

Add GEO_AREA support to gooddata-dbt

Author: hkad98

packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py

@@ -21,6 +21,7 @@ class GoodDataLabelType(Enum):
     HYPERLINK = "HYPERLINK"
     GEO_LATITUDE = "GEO_LATITUDE"
     GEO_LONGITUDE = "GEO_LONGITUDE"
+    GEO_AREA = "GEO_AREA"
 
 
 class GoodDataSortDirection(Enum):

packages/gooddata-dbt/src/gooddata_dbt/dbt/metrics.py

@@ -1,7 +1,6 @@
 # (C) 2023 GoodData Corporation
 import json
 import re
-from typing import Optional
 
 import attrs
 from gooddata_sdk import CatalogDeclarativeMetric, CatalogDeclarativeModel
@@ -26,8 +25,8 @@
 
 @attrs.define(auto_attribs=True, kw_only=True)
 class DbtModelMetaGoodDataMetricProps(Base):
-    model_id: Optional[str] = None
-    format: Optional[str] = None
+    model_id: str | None = None
+    format: str | None = None
 
 
 @attrs.define(auto_attribs=True, kw_only=True)
@@ -49,11 +48,11 @@ class DbtModelMetric(DbtModelBase):
     model: str
     calculation_method: str
     expression: str
-    filters: Optional[list[DbtModelMetricFilter]] = None
+    filters: list[DbtModelMetricFilter] | None = None
 
 
 class DbtModelMetrics:
-    def __init__(self, model_ids: Optional[list[str]], ldm: CatalogDeclarativeModel) -> None:
+    def __init__(self, model_ids: list[str] | None, ldm: CatalogDeclarativeModel) -> None:
         self.model_ids = model_ids
         self.ldm = ldm
         with open(DBT_PATH_TO_MANIFEST) as fp:
@@ -104,7 +103,7 @@ def get_entity_type(self, table_name: str, expression_entity: str) -> str:
         else:
             raise Exception(f"Unsupported entity type {table_name=} {expression_entity=}")
 
-    def make_entity_id(self, table_name: str, token: str) -> Optional[str]:
+    def make_entity_id(self, table_name: str, token: str) -> str | None:
         entity_type = self.get_entity_type(table_name, token)
         if not entity_type:
             return None
@@ -125,7 +124,7 @@ def resolve_entities_in_expression(self, expression: str, table_name: str) -> st
             result_tokens.append(entity_id or token)
         return " ".join(result_tokens)
 
-    def make_gooddata_filter(self, table_name: str, dbt_filters: Optional[list[DbtModelMetricFilter]] = None) -> str:
+    def make_gooddata_filter(self, table_name: str, dbt_filters: list[DbtModelMetricFilter] | None = None) -> str:
         # TODO - Quite naive implementation
         #    e.g. missing polishing of values (e.g. SQL vs MAQL enclosers)
         gd_maql_filters = []

packages/gooddata-dbt/src/gooddata_dbt/dbt/profiles.py

@@ -2,7 +2,7 @@
 import argparse
 import os
 import re
-from typing import Optional, Union
+from typing import Union
 from urllib.parse import quote_plus
 
 import attrs
@@ -97,7 +97,7 @@ class DbtOutputSnowflake(Base):
     database: str
     warehouse: str
     schema: str
-    query_tag: Optional[str] = None
+    query_tag: str | None = None
 
     def to_gooddata(self, data_source_id: str, schema_name: str) -> CatalogDataSourceSnowflake:
         return CatalogDataSourceSnowflake(
@@ -222,7 +222,7 @@ def inject_env_vars(output_def: dict) -> None:
             # else do nothing, real value seems to be stored in dbt profile
 
     @staticmethod
-    def to_data_class(output: str, output_def: dict) -> Optional[DbtOutput]:
+    def to_data_class(output: str, output_def: dict) -> DbtOutput | None:
         db_type = output_def["type"]
         if db_type == "postgres":
             return DbtOutputPostgreSQL.from_dict({"name": output, **output_def})

packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py

@@ -3,7 +3,7 @@
 import json
 import re
 from pathlib import Path
-from typing import Optional, Union
+from typing import Union
 
 import attrs
 from gooddata_sdk import CatalogDeclarativeColumn, CatalogDeclarativeTable, CatalogDeclarativeTables
@@ -27,22 +27,23 @@
 
 @attrs.define(auto_attribs=True, kw_only=True)
 class DbtModelMetaGoodDataTableProps(Base):
-    model_id: Optional[str] = None
+    model_id: str | None = None
 
 
 @attrs.define(auto_attribs=True, kw_only=True)
 class DbtModelMetaGoodDataColumnProps(Base):
-    id: Optional[str] = None
-    ldm_type: Optional[GoodDataLdmType] = None
-    referenced_table: Optional[str] = None
-    label_type: Optional[GoodDataLabelType] = None
-    attribute_column: Optional[str] = None
-    sort_column: Optional[str] = None
-    sort_direction: Optional[GoodDataSortDirection] = None
-    default_view: Optional[bool] = None
+    id: str | None = None
+    ldm_type: GoodDataLdmType | None = None
+    referenced_table: str | None = None
+    label_type: GoodDataLabelType | None = None
+    attribute_column: str | None = None
+    sort_column: str | None = None
+    sort_direction: GoodDataSortDirection | None = None
+    default_view: bool | None = None
+    geo_area_config: dict[str, dict[str, str]] | None = None
 
     @property
-    def gooddata_ref_table_ldm_id(self) -> Optional[str]:
+    def gooddata_ref_table_ldm_id(self) -> str | None:
         if self.referenced_table:
             return self.referenced_table.lower()
         return None
@@ -73,7 +74,7 @@ class DbtModelBase(Base):
     tags: list[str]
     # If 2+ references point to the same table, the table plays multiple roles,
     # it must be generated as multiple datasets
-    role_name: Optional[str] = None
+    role_name: str | None = None
 
     # TODO - duplicate of backend logic.
     # Solution: use result of generateLdm as a master template, and override based on dbt metadata only if necessary
@@ -120,7 +121,7 @@ def upper_case_name(self) -> None:
 
 @attrs.define(auto_attribs=True, kw_only=True)
 class DbtModelColumn(DbtModelBase):
-    data_type: Optional[str]
+    data_type: str | None
     meta: DbtModelMetaGoodDataColumn = attrs.field(factory=DbtModelMetaGoodDataColumn)
 
     # Enable to override LDM ID for LDM entities derived from columns (attributes, ...)
@@ -130,21 +131,21 @@ def ldm_id(self) -> str:
         return self.meta.gooddata.id or self.gooddata_ldm_id
 
     @property
-    def ldm_type(self) -> Optional[str]:
+    def ldm_type(self) -> str | None:
         if self.meta.gooddata.ldm_type is None:
             return None
         else:
             return self.meta.gooddata.ldm_type.value
 
     @property
-    def label_type(self) -> Optional[str]:
+    def label_type(self) -> str | None:
         if self.meta.gooddata.label_type is None:
             return None
         else:
             return self.meta.gooddata.label_type.value
 
     @property
-    def sort_direction(self) -> Optional[str]:
+    def sort_direction(self) -> str | None:
         if self.meta.gooddata.sort_direction is None:
             return None
         else:
@@ -387,22 +388,23 @@ def make_facts(table: DbtModelTable) -> list[dict]:
         return facts
 
     @staticmethod
-    def make_labels(table: DbtModelTable, attribute_column: DbtModelColumn) -> tuple[list[dict], Optional[dict]]:
+    def make_labels(table: DbtModelTable, attribute_column: DbtModelColumn) -> tuple[list[dict], dict | None]:
         labels = []
         default_view = None
         for column in table.columns.values():
             if column.gooddata_is_label(attribute_column.name):
-                labels.append(
-                    {
-                        "id": column.ldm_id,
-                        "title": column.gooddata_ldm_title,
-                        "description": column.gooddata_ldm_description,
-                        "source_column": column.name,
-                        "source_column_data_type": column.data_type,
-                        "value_type": column.label_type,
-                        "tags": [table.gooddata_ldm_title] + column.tags,
-                    }
-                )
+                label_dict: dict = {
+                    "id": column.ldm_id,
+                    "title": column.gooddata_ldm_title,
+                    "description": column.gooddata_ldm_description,
+                    "source_column": column.name,
+                    "source_column_data_type": column.data_type,
+                    "value_type": column.label_type,
+                    "tags": [table.gooddata_ldm_title] + column.tags,
+                }
+                if column.meta.gooddata.geo_area_config is not None:
+                    label_dict["geo_area_config"] = column.meta.gooddata.geo_area_config
+                labels.append(label_dict)
                 if column.meta.gooddata.default_view:
                     default_view = {
                         "id": column.ldm_id,
@@ -507,7 +509,7 @@ def populate_role_playing_tables(tables: list[DbtModelTable], role_playing_table
                 result.append(table)
         return result
 
-    def make_declarative_datasets(self, data_source_id: str, model_ids: Optional[list[str]]) -> dict:
+    def make_declarative_datasets(self, data_source_id: str, model_ids: list[str] | None) -> dict:
         result: dict[str, list] = {"datasets": [], "date_instances": []}
         model_tables = [t for t in self.tables if not model_ids or t.meta.gooddata.model_id in model_ids]
         role_playing_tables = self.find_role_playing_tables(model_tables)
@@ -517,7 +519,7 @@ def make_declarative_datasets(self, data_source_id: str, model_ids: Optional[lis
             result = self.make_dataset(data_source_id, table, role_playing_tables, result)
         return result
 
-    def get_entity_type(self, table_name: str, column_name: str) -> Optional[str]:
+    def get_entity_type(self, table_name: str, column_name: str) -> str | None:
         comp_table_name = table_name
         if self.upper_case:
             comp_table_name = table_name.upper()

packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py

@@ -5,7 +5,6 @@
 from asyncio import Semaphore
 from pathlib import Path
 from time import time
-from typing import Optional
 
 import tabulate
 import yaml
@@ -36,7 +35,7 @@ def generate_and_put_ldm(
     data_source_id: str,
     workspace_id: str,
     dbt_tables: DbtModelTables,
-    model_ids: Optional[list[str]],
+    model_ids: list[str] | None,
 ) -> None:
     scan_request = CatalogScanModelRequest(scan_tables=True, scan_views=True)
     logger.info(f"Scan data source {data_source_id=}")
@@ -68,7 +67,7 @@ def deploy_ldm(
     args: Namespace,
     all_model_ids: list[str],
     sdk_wrapper: GoodDataSdkWrapper,
-    model_ids: Optional[list[str]],
+    model_ids: list[str] | None,
     workspace_id: str,
 ) -> None:
     logger.info("Generate and put LDM")
@@ -186,7 +185,7 @@ async def test_visualizations(
     logger: logging.Logger,
     sdk_wrapper: GoodDataSdkWrapper,
     workspace_id: str,
-    skip_tests: Optional[list[str]],
+    skip_tests: list[str] | None,
     test_visualizations_parallelism: int = 1,
 ) -> None:
     start = time()
@@ -334,7 +333,7 @@ def process_organization(
     logger: logging.Logger,
     sdk_wrapper: GoodDataSdkWrapper,
     gd_config: GoodDataConfig,
-    organization: Optional[GoodDataConfigOrganization] = None,
+    organization: GoodDataConfigOrganization | None = None,
 ) -> None:
     if args.method == "upload_notification":
         dbt_profiles = DbtProfiles(args)

packages/gooddata-dbt/src/gooddata_dbt/gooddata/config.py

@@ -1,5 +1,4 @@
 # (C) 2023 GoodData Corporation
-from typing import Optional
 
 import attr
 import attrs
@@ -37,8 +36,8 @@ class GoodDataConfigProduct(Base):
     name: str
     environment_setup_id: str
     model_ids: list[str] = attr.field(factory=list)
-    localization: Optional[GoodDataConfigLocalization] = None
-    skip_tests: Optional[list[str]] = None
+    localization: GoodDataConfigLocalization | None = None
+    skip_tests: list[str] | None = None
 
 
 @attrs.define(auto_attribs=True, kw_only=True)
@@ -49,7 +48,7 @@ class GoodDataConfigOrganization(Base):
 
 @attrs.define(auto_attribs=True, kw_only=True)
 class GoodDataGlobalConfig(Base):
-    test_visualizations_parallelism: Optional[int] = 1
+    test_visualizations_parallelism: int | None = 1
 
 
 @attrs.define(auto_attribs=True, kw_only=True)

packages/gooddata-dbt/src/gooddata_dbt/sdk_wrapper.py

@@ -1,7 +1,6 @@
 # (C) 2023 GoodData Corporation
 import argparse
 from logging import Logger
-from typing import Optional
 
 from gooddata_sdk import GoodDataSdk
 
@@ -11,7 +10,7 @@
 class GoodDataSdkWrapper:
     # Timeout=600 because supporting waiting for GoodData services to start
     def __init__(
-        self, args: argparse.Namespace, logger: Logger, profile: Optional[str] = None, timeout: int = 600
+        self, args: argparse.Namespace, logger: Logger, profile: str | None = None, timeout: int = 600
     ) -> None:
         self.args = args
         self.logger = logger
@@ -22,7 +21,7 @@ def __init__(
         if not self.args.dry_run:
             self.wait_for_gooddata_is_up(self.timeout)
 
-    def get_host_from_sdk(self) -> Optional[str]:
+    def get_host_from_sdk(self) -> str | None:
         # TODO - make _hostname public in gooddata_sdk
         return self.sdk.client._hostname
 
@@ -54,7 +53,7 @@ def wait_for_gooddata_is_up(self, timeout: int) -> None:
         self.sdk.support.wait_till_available(timeout=timeout)
         self.logger.info(f"Host {host} is up")
 
-    def pre_cache_visualizations(self, workspaces: Optional[list] = None) -> None:
+    def pre_cache_visualizations(self, workspaces: list | None = None) -> None:
         if not workspaces:
             workspaces = [w.id for w in self.sdk.catalog_workspace.list_workspaces()]
         for workspace_id in workspaces:

packages/gooddata-dbt/tests/resources/dbt_target/manifest.json

@@ -1860,6 +1860,25 @@
           "data_type": null,
           "quote": null,
           "tags": []
+        },
+        "country": {
+          "name": "country",
+          "description": "",
+          "meta": {
+            "gooddata": {
+              "ldm_type": "label",
+              "label_type": "GEO_AREA",
+              "attribute_column": "code",
+              "geo_area_config": {
+                "collection": {
+                  "id": "countries"
+                }
+              }
+            }
+          },
+          "data_type": null,
+          "quote": null,
+          "tags": []
         }
       },
       "meta": {

packages/gooddata-dbt/tests/resources/gooddata_layouts/pdm/airports.yaml

@@ -21,6 +21,9 @@ columns:
   - dataType: STRING
     isPrimaryKey: false
     name: name
+  - dataType: STRING
+    isPrimaryKey: false
+    name: country
   - dataType: STRING
     isPrimaryKey: false
     name: state