livevalidator/databricks.yml.example at main · databricks-solutions/livevalidator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
bundle:
  name: LiveValidator
  engine: direct

include:
  - resources/*.yml

workspace:
  root_path: /Workspace/LiveValidator

variables:
  serverless_version:
    description: The serverless environment version
    default: 5

resources:

  # Web app
  apps:
    live_validator:
      name: live-validator
      description: Compare tables or query results across systems
      source_code_path: src/app
      permissions:
        - level: CAN_USE
          group_name: users
      resources:
        - name: postgres
          database:
            database_name: databricks_postgres
            instance_name: ${resources.database_instances.live_validator_database.name}
            permission: CAN_CONNECT_AND_CREATE
        - name: run-validation
          job:
            id: ${resources.jobs.run_validation.id}
            permission: 'CAN_MANAGE_RUN'
        - name: run-validation-serverless
          job:
            id: ${resources.jobs.run_validation_serverless.id}
            permission: 'CAN_MANAGE_RUN'
        - name: test-connection
          job:
            id: ${resources.jobs.test_connection.id}
            permission: 'CAN_MANAGE_RUN'
        - name: test-connection-serverless
          job:
            id: ${resources.jobs.test_connection_serverless.id}
            permission: 'CAN_MANAGE_RUN'
        - name: fetch-lineage
          job:
            id: ${resources.jobs.fetch_lineage.id}
            permission: 'CAN_MANAGE_RUN'
        - name: db_user
          secret:
            key: db_user
            permission: "READ"
            scope: livevalidator
        - name: db_password
          secret:
            key: db_password
            permission: "READ"
            scope: livevalidator

  # LakeBase
  database_instances:
    live_validator_database:
      name: live-validator
      enable_pg_native_login: true
      capacity: CU_1
  database_catalogs:
    live_validator_catalog:
      database_instance_name: ${resources.database_instances.live_validator_database.name}
      name: live_validator
      database_name: databricks_postgres
      create_database_if_not_exists: true

  # Classic cluster, required for Classic Compute sources
  clusters:
    live_validator_cluster:
      cluster_name: live-validator
      node_type_id: i3.2xlarge
      driver_node_type_id: i3.2xlarge
      autoscale:
        min_workers: 1
        max_workers: 8
      aws_attributes:
        availability: ON_DEMAND
      spark_version: 17.3.x-scala2.13
      autotermination_minutes: 0
      enable_local_disk_encryption: false
      data_security_mode: USER_ISOLATION
      runtime_engine: PHOTON

  # Required for Serverless: catalog for temp results
  catalogs:
    live_validator_data:
      name: live_validator_data
      comment: "Store LiveValidator information as Delta"
      grants:
        - principal: account users
          privileges:
            - USE CATALOG
            - SELECT
  schemas:
    entities:
      name: entities
      catalog_name: ${resources.catalogs.live_validator_data.name}
      comment: "Store point-in-time entity data"

targets:

  permissions:
  - user_name: <deployer_email>
    level: CAN_MANAGE

  migr-prod:
    default: true
    mode: production
    workspace:
      host: https://your-deployment-workspace.cloud.databricks.com/
      root_path: /Workspace/LiveValidator
    permissions:
      - level: CAN_MANAGE
        group_name: 1dp-migration-prd-admin
      - level: CAN_RUN
        group_name: users

    resources:
      clusters:
        live_validator_cluster:
          spark_conf:
            spark.your.additional.config: 'true'
          permissions:
            - group_name: users
              level: CAN_ATTACH_TO
            - group_name: 1dp-migration-prd-admin
              level: CAN_MANAGE
      jobs:
        run_validation:
          tasks:
            - task_key: validate
              # add all libraries to UC allowlist (!!!)
              libraries:
                # Both of these required for Teradata
                - maven:
                    coordinates: com.teradata.jdbc:terajdbc:20.00.00.51
                - pypi:
                    package: teradatasql
                # Required for Oracle
                - maven:
                    coordinates: com.oracle.database.jdbc:ojdbc11:23.5.0.24.07
          permissions:
            - level: CAN_MANAGE
              group_name: 1dp-migration-prd-admin
        run_validation_serverless:
          permissions:
            - level: CAN_MANAGE
              group_name: 1dp-migration-prd-admin