Skip to content

Commit 53a10e0

Browse files
drgrootdrgroot
committed
feat: initial working cut of the provisioner
for ticket #1
1 parent 0b44508 commit 53a10e0

File tree

15 files changed

+424
-680
lines changed

15 files changed

+424
-680
lines changed

‎.env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
ACCESS_KEY_ID=user
2+
ACCESS_SECRET_ACCESS_KEY=password
3+
DEFAULT_BUCKET=servc

‎.github/workflows/test.yml‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ name: 'Unit Tests'
22
on:
33
- push
44

5+
env:
6+
ACCESS_KEY_ID: user
7+
ACCESS_SECRET_ACCESS_KEY: password
8+
DEFAULT_BUCKET: mylake
9+
510
jobs:
611
unittest:
712
runs-on: ubuntu-latest
@@ -14,6 +19,35 @@ jobs:
1419
- 3.13
1520

1621
services:
22+
catalog:
23+
image: tabulario/iceberg-rest
24+
env:
25+
AWS_ACCESS_KEY_ID: ${{ env.ACCESS_KEY_ID }}
26+
AWS_SECRET_ACCESS_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
27+
AWS_REGION: us-east-1
28+
WAREHOUSE: s3://${{ env.DEFAULT_BUCKET }}
29+
CATALOG_WAREHOUSE: s3://${{ env.DEFAULT_BUCKET }}
30+
CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
31+
CATALOG_S3_ENDPOINT: http://s3:9000
32+
CATALOG_S3_PATH__STYLE__ACCESS: "true"
33+
CATALOG_DEFAULT__NAMESPACE: default
34+
CATALOG_JDBC_USER: user
35+
CATALOG_JDBC_PASSWORD: pass
36+
CATALOG_URI: jdbc:sqlite:file:/tmp/pyiceberg_catalog.db
37+
ports:
38+
- 8181:8181
39+
40+
s3:
41+
image: bitnami/minio:latest
42+
env:
43+
MINIO_ROOT_USER: ${{ env.ACCESS_KEY_ID }}
44+
MINIO_ROOT_PASSWORD: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
45+
MINIO_ACCESS_KEY: ${{ env.ACCESS_KEY_ID }}
46+
MINIO_SECRET_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
47+
ports:
48+
- 9000:9000
49+
- 9001:9001
50+
1751
rabbitmq:
1852
image: rabbitmq
1953
env:
@@ -59,13 +93,34 @@ jobs:
5993
- name: Type check
6094
run: mypy provisioner --check-untyped-defs
6195

96+
- name: Install minioclient
97+
env:
98+
MC_HOST_servc: http://${{ env.ACCESS_KEY_ID }}:${{ env.ACCESS_SECRET_ACCESS_KEY }}@${{ github.server_url != 'https://github.com' && 's3' || 'localhost' }}:9000
99+
run: |
100+
sudo apt-get update && sudo apt-get install -y wget
101+
102+
wget https://dl.min.io/client/mc/release/linux-amd64/mc
103+
chmod +x mc
104+
sudo mv mc /usr/local/bin
105+
106+
mc mb servc/${{ env.DEFAULT_BUCKET }}
107+
62108
- name: Run tests
63109
env:
110+
CONF__BLOBPROVISIONER__S3_ACCESS_KEY_ID: ${{ env.ACCESS_KEY_ID }}
111+
CONF__BLOBPROVISIONER__S3_SECRET_ACCESS_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
112+
CONF__BLOBPROVISIONER__S3_ENDPOINT: http://${{ github.server_url != 'https://github.com' && 's3' || 'localhost' }}:9000
113+
CONF__BLOBPROVISIONER__S3_BUCKET: ${{ env.DEFAULT_BUCKET }}
114+
CONF__BLOBPROVISIONER__CATALOG_URI: http://${{ github.server_url != 'https://github.com' && 'catalog' || 'localhost' }}:8181
115+
CONF__LAKE__DATABASE: servc
64116
CONF__LAKE__CATALOG_NAME: default
65-
CONF__LAKE__CATALOG_PROPERTIES__TYPE: sql
66-
CONF__LAKE__CATALOG_PROPERTIES__URI: sqlite:////tmp/lake.db
117+
CONF__LAKE__CATALOG_PROPERTIES__TYPE: rest
118+
CONF__LAKE__CATALOG_PROPERTIES__URI: http://${{ github.server_url != 'https://github.com' && 'catalog' || 'localhost' }}:8181
67119
CONF__LAKE__CATALOG_PROPERTIES__INIT_CATALOG_TABLES: true
68-
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: file:///tmp/warehouse
120+
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: s3://${{ env.DEFAULT_BUCKET }}
121+
CONF__LAKE__CATALOG_PROPERTIES__S3__ACCESS_KEY_ID: ${{ env.ACCESS_KEY_ID }}
122+
CONF__LAKE__CATALOG_PROPERTIES__S3__SECRET_ACCESS_KEY: ${{ env.ACCESS_SECRET_ACCESS_KEY }}
123+
CONF__LAKE__CATALOG_PROPERTIES__S3__ENDPOINT: http://${{ github.server_url != 'https://github.com' && 's3' || 'localhost' }}:9000
69124
TEST_QUEUE: test
70125
CACHE_URL: redis://${{ github.server_url != 'https://github.com' && 'redis' || 'localhost' }}
71126
BUS_URL: amqp://guest:guest@${{ github.server_url != 'https://github.com' && 'rabbitmq' || 'localhost' }}

‎Dockerfile.dev‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM minio/mc AS minio
2+
3+
FROM mcr.microsoft.com/vscode/devcontainers/python
4+
5+
COPY --from=minio /usr/bin/mc /usr/bin/mc

‎docker-compose.yml‎

Lines changed: 52 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,65 @@
1-
version: "3.8"
2-
31
services:
42
python:
5-
image: mcr.microsoft.com/vscode/devcontainers/python
3+
image: python-minio
4+
build:
5+
context: .
6+
dockerfile: Dockerfile.dev
67
volumes:
78
- .:/workspace:cached
89
- /Users/yusuf/Documents/bin:/custom_bin:ro
910
command: sleep infinity
1011
environment:
11-
CACHE_URL: redis://redis:6379/0
12-
BUS_URL: amqp://rabbitmq:rabbitmq@rabbitmq/
12+
CONF__BLOBPROVISIONER__S3_ACCESS_KEY_ID: ${ACCESS_KEY_ID}
13+
CONF__BLOBPROVISIONER__S3_SECRET_ACCESS_KEY: ${ACCESS_SECRET_ACCESS_KEY}
14+
CONF__BLOBPROVISIONER__S3_ENDPOINT: http://s3:9000
15+
CONF__BLOBPROVISIONER__S3_BUCKET: ${DEFAULT_BUCKET}
16+
CONF__BLOBPROVISIONER__CATALOG_URI: http://catalog:8181
17+
CONF__LAKE__DATABASE: servc
1318
CONF__LAKE__CATALOG_NAME: default
14-
CONF__LAKE__CATALOG_PROPERTIES__TYPE: sql
15-
CONF__LAKE__CATALOG_PROPERTIES__URI: sqlite:////tmp/lake.db
19+
CONF__LAKE__CATALOG_PROPERTIES__TYPE: rest
20+
CONF__LAKE__CATALOG_PROPERTIES__URI: http://catalog:8181
1621
CONF__LAKE__CATALOG_PROPERTIES__INIT_CATALOG_TABLES: true
17-
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: file:///tmp/warehouse
22+
CONF__LAKE__CATALOG_PROPERTIES__WAREHOUSE: s3://${DEFAULT_BUCKET}
23+
CONF__LAKE__CATALOG_PROPERTIES__S3__ACCESS_KEY_ID: ${ACCESS_KEY_ID}
24+
CONF__LAKE__CATALOG_PROPERTIES__S3__SECRET_ACCESS_KEY: ${ACCESS_SECRET_ACCESS_KEY}
25+
CONF__LAKE__CATALOG_PROPERTIES__S3__ENDPOINT: http://s3:9000
26+
links:
27+
- catalog
28+
- s3
1829

19-
redis:
20-
image: redis
21-
restart: unless-stopped
22-
expose:
23-
- "6379"
24-
ports:
25-
- "6379:6379"
30+
catalog:
31+
image: tabulario/iceberg-rest
32+
environment:
33+
AWS_ACCESS_KEY_ID: ${ACCESS_KEY_ID}
34+
AWS_SECRET_ACCESS_KEY: ${ACCESS_SECRET_ACCESS_KEY}
35+
AWS_REGION: us-east-1
36+
WAREHOUSE: s3://${DEFAULT_BUCKET}
37+
CATALOG_WAREHOUSE: s3://${DEFAULT_BUCKET}
38+
CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
39+
CATALOG_S3_ENDPOINT: http://s3:9000
40+
CATALOG_S3_PATH__STYLE__ACCESS: true
41+
CATALOG_DEFAULT__NAMESPACE: default
42+
CATALOG_JDBC_USER: user
43+
CATALOG_JDBC_PASSWORD: pass
44+
CATALOG_URI: jdbc:sqlite:file:/tmp/pyiceberg_catalog.db
2645

27-
rabbitmq:
28-
image: rabbitmq:4-management
46+
s3:
47+
image: quay.io/minio/minio
48+
command: server /data --console-address ":9001"
2949
environment:
30-
RABBITMQ_DEFAULT_USER: rabbitmq
31-
RABBITMQ_DEFAULT_PASS: rabbitmq
32-
RABBITMQ_DEFAULT_VHOST: /
50+
MINIO_ROOT_USER: ${ACCESS_KEY_ID}
51+
MINIO_ROOT_PASSWORD: ${ACCESS_SECRET_ACCESS_KEY}
3352
ports:
34-
- "5672:5672"
35-
- "15672:15672"
36-
expose:
37-
- "5672"
38-
- "15672"
53+
- "9000:9000"
54+
- "9001:9001"
55+
56+
create_bucket:
57+
restart: "no"
58+
image: minio/mc
59+
entrypoint: sh
60+
command: -c "sleep 10s && mc alias set minio http://s3:9000 ${ACCESS_KEY_ID} ${ACCESS_SECRET_ACCESS_KEY} && mc mb minio/${DEFAULT_BUCKET}"
61+
environment:
62+
MINIO_ROOT_USER: ${ACCESS_KEY_ID}
63+
MINIO_ROOT_PASSWORD: ${ACCESS_SECRET_ACCESS_KEY}
64+
depends_on:
65+
- s3

‎main.py‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,19 @@
11
#!/usr/bin/env python
22

33
from servc.server import start_server
4+
from servc.svc.com.worker.types import RESOLVER_MAPPING
5+
from servc_typings.domains.orchestrator import Events
46

5-
from orchestrator.domains.add_dag import add_dag
6-
from orchestrator.domains.trigger import trigger
7-
from orchestrator.domains.update_status import update
8-
from orchestrator.config import QUEUE_NAME
7+
from provisioner.events.dag_updated import registerDag
8+
9+
eventResolver: RESOLVER_MAPPING = {}
10+
eventResolver[Events.DAG_UPDATED.value] = registerDag
911

1012

1113
def main():
1214
return start_server(
13-
resolver={
14-
"add_dag": add_dag,
15-
"trigger": trigger,
16-
"update_status": update,
17-
},
18-
route=QUEUE_NAME,
15+
resolver={},
16+
eventResolver=eventResolver,
1917
)
2018

2119

‎provisioner/__init__.py‎

Whitespace-only changes.

‎provisioner/com/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)