From 373a4b819860ce7e9af16d277dae7337f943b5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 31 Oct 2025 11:24:43 +0300 Subject: [PATCH] [DOP-29496] Load settings from config.yml --- .env.docker | 58 ---- .env.local | 58 ---- .github/workflows/release.yaml | 1 - .github/workflows/unit-test.yml | 1 - CONTRIBUTING.rst | 5 +- Makefile | 1 - config.docker.yml | 68 +++++ config.yml | 71 +++++ docker-compose.test.yml | 11 +- docker-compose.yml | 16 +- docker/entrypoint_server.sh | 7 +- .../changelog/next_release/289.breaking.1.rst | 2 + .../changelog/next_release/289.breaking.2.rst | 1 + .../changelog/next_release/289.breaking.3.rst | 1 + docs/conf.py | 2 +- docs/reference/broker/index.rst | 12 +- docs/reference/database/index.rst | 25 +- docs/reference/frontend/configuration.rst | 23 +- docs/reference/frontend/index.rst | 12 +- .../scheduler/configuration/logging.rst | 2 +- docs/reference/scheduler/index.rst | 12 +- docs/reference/scheduler/install.rst | 58 ---- .../auth/keycloak/local_installation.rst | 130 ++++---- docs/reference/server/configuration/debug.rst | 33 +- .../server/configuration/logging.rst | 2 +- docs/reference/server/index.rst | 12 +- .../server/manage_superusers_cli.rst | 21 +- .../worker/configuration/logging.rst | 2 +- .../reference/worker/create_spark_session.rst | 6 +- docs/reference/worker/index.rst | 12 +- docs/reference/worker/log_url.rst | 6 +- poetry.lock | 18 +- pyproject.toml | 3 +- syncmaster/__init__.py | 2 +- syncmaster/db/migrations/env.py | 15 +- syncmaster/scheduler/__main__.py | 4 +- syncmaster/scheduler/settings/__init__.py | 54 ++-- syncmaster/schemas/v1/transfers/__init__.py | 2 +- syncmaster/server/__init__.py | 5 +- syncmaster/server/middlewares/__init__.py | 4 - .../server/scripts/manage_superusers.py | 33 +- syncmaster/server/settings/__init__.py | 45 +-- syncmaster/server/settings/auth/__init__.py | 14 +- syncmaster/server/settings/auth/dummy.py | 9 +- syncmaster/server/settings/auth/jwt.py | 9 +- syncmaster/server/settings/auth/keycloak.py | 25 +- .../server/settings/auth/oauth2_gateway.py | 19 +- syncmaster/server/settings/server/__init__.py | 20 +- syncmaster/server/settings/server/cors.py | 51 ++-- .../server/settings/server/monitoring.py | 19 +- syncmaster/server/settings/server/openapi.py | 66 ++-- .../server/settings/server/request_id.py | 12 +- syncmaster/server/settings/server/session.py | 34 ++- .../server/settings/server/static_files.py | 15 +- syncmaster/settings/__init__.py | 7 +- syncmaster/settings/base.py | 37 +++ syncmaster/settings/broker.py | 11 +- syncmaster/settings/credentials.py | 7 +- syncmaster/settings/database.py | 11 +- syncmaster/settings/log/__init__.py | 110 ------- syncmaster/settings/log/colored.yml | 57 ---- syncmaster/settings/log/json.yml | 56 ---- syncmaster/settings/log/plain.yml | 57 ---- syncmaster/settings/logging.py | 288 ++++++++++++++++++ syncmaster/worker/settings/__init__.py | 52 ++-- syncmaster/worker/settings/hwm_store.py | 18 +- syncmaster/worker/transfer.py | 4 +- .../test_auth/auth_fixtures/__init__.py | 1 + .../auth_fixtures/keycloak_fixture.py | 1 + ...test_auth_keycloak.py => test_keycloak.py} | 88 ++---- .../test_auth/test_oauth2_gateway.py | 40 ++- 71 files changed, 1092 insertions(+), 902 deletions(-) delete mode 100644 .env.docker delete mode 100644 .env.local create mode 100644 config.docker.yml create mode 100644 config.yml create mode 100644 docs/changelog/next_release/289.breaking.1.rst create mode 100644 docs/changelog/next_release/289.breaking.2.rst create mode 100644 docs/changelog/next_release/289.breaking.3.rst delete mode 100644 docs/reference/scheduler/install.rst create mode 100644 syncmaster/settings/base.py delete mode 100644 syncmaster/settings/log/__init__.py delete mode 100644 syncmaster/settings/log/colored.yml delete mode 100644 syncmaster/settings/log/json.yml delete mode 100644 syncmaster/settings/log/plain.yml create mode 100644 syncmaster/settings/logging.py rename tests/test_unit/test_auth/{test_auth_keycloak.py => test_keycloak.py} (77%) diff --git a/.env.docker b/.env.docker deleted file mode 100644 index ebd182a5..00000000 --- a/.env.docker +++ /dev/null @@ -1,58 +0,0 @@ -TZ=UTC -ENV=LOCAL - -# Logging options -SYNCMASTER__LOGGING__SETUP=True -SYNCMASTER__LOGGING__PRESET=colored - -# Common DB options -SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster - -# Encrypt / Decrypt credentials data using this Fernet key. -# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!! -SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= - -# Common RabbitMQ options -SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672 - -# Server options -SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string -# !!! NEVER USE ON PRODUCTION !!! -SYNCMASTER__SERVER__DEBUG=true - -# Keycloak Auth -#SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider -SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 -SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created -SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created -SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak -SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:3000/auth/callback -SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email -SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False - -# Dummy Auth -SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider -SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string - -# Scheduler options -SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 - -# Worker options -SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }} -SYNCMASTER__HWM_STORE__ENABLED=true -SYNCMASTER__HWM_STORE__TYPE=horizon -SYNCMASTER__HWM_STORE__URL=http://horizon:8000 -SYNCMASTER__HWM_STORE__NAMESPACE=syncmaster_namespace -SYNCMASTER__HWM_STORE__USER=admin -SYNCMASTER__HWM_STORE__PASSWORD=123UsedForTestOnly@! - -# Frontend options -SYNCMASTER__UI__API_BROWSER_URL=http://localhost:8000 - -# Cors -SYNCMASTER__SERVER__CORS__ENABLED=True -SYNCMASTER__SERVER__CORS__ALLOW_ORIGINS=["http://localhost:3000"] -SYNCMASTER__SERVER__CORS__ALLOW_CREDENTIALS=True -SYNCMASTER__SERVER__CORS__ALLOW_METHODS=["*"] -SYNCMASTER__SERVER__CORS__ALLOW_HEADERS=["*"] -SYNCMASTER__SERVER__CORS__EXPOSE_HEADERS=["X-Request-ID","Location","Access-Control-Allow-Credentials"] diff --git a/.env.local b/.env.local deleted file mode 100644 index 9ec39027..00000000 --- a/.env.local +++ /dev/null @@ -1,58 +0,0 @@ -export TZ=UTC -export ENV=LOCAL - -# Logging options -export SYNCMASTER__LOGGING__SETUP=True -export SYNCMASTER__LOGGING__PRESET=colored - -# Common DB options -export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster - -# Encrypt / Decrypt credentials data using this Fernet key. -# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!! -export SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= - -# Common RabbitMQ options -export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672 - -# Server options -export SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string -# !!! NEVER USE ON PRODUCTION !!! -export SYNCMASTER__SERVER__DEBUG=true - -# Keycloak Auth -#export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider -export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://localhost:8080 -export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created -export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created -export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak -export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:3000/auth/callback -export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email -export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False - -# Dummy Auth -export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider -export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string - -# Scheduler options -export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 - -# Worker options -export SYNCMASTER__WORKER__LOG_URL_TEMPLATE="https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}" -export SYNCMASTER__HWM_STORE__ENABLED=true -export SYNCMASTER__HWM_STORE__TYPE=horizon -export SYNCMASTER__HWM_STORE__URL=http://localhost:8020 -export SYNCMASTER__HWM_STORE__NAMESPACE=syncmaster_namespace -export SYNCMASTER__HWM_STORE__USER=admin -export SYNCMASTER__HWM_STORE__PASSWORD=123UsedForTestOnly@! - -# Frontend options -export SYNCMASTER__UI__API_BROWSER_URL=http://localhost:8000 - -# Cors -export 'SYNCMASTER__SERVER__CORS__ENABLED=True' -export 'SYNCMASTER__SERVER__CORS__ALLOW_ORIGINS=["http://localhost:3000"]' -export 'SYNCMASTER__SERVER__CORS__ALLOW_CREDENTIALS=True' -export 'SYNCMASTER__SERVER__CORS__ALLOW_METHODS=["*"]' -export 'SYNCMASTER__SERVER__CORS__ALLOW_HEADERS=["*"]' -export 'SYNCMASTER__SERVER__CORS__EXPOSE_HEADERS=["X-Request-ID","Location","Access-Control-Allow-Credentials"]' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 95bfe465..fc5e78e0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -54,7 +54,6 @@ jobs: - name: Generate OpenAPI Schema run: | - source .env.local poetry run python -m syncmaster.server.scripts.export_openapi_schema docs/_static/openapi.json - name: Fix logo in Readme diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 2763261d..54a965ff 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -65,7 +65,6 @@ jobs: - name: Run Unit Tests run: | - source .env.local source .env.local.test poetry run coverage run -m pytest -vvv -s -k "test_unit or test_database" diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9eb850aa..0a2a0104 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -101,7 +101,7 @@ Then start development server: And open http://localhost:8000/docs -Settings are stored in ``.env.local`` file. +Settings are stored in ``config.yml`` file. To start development worker, open a new terminal window/tab, and run: @@ -181,6 +181,7 @@ To run specific integration tests: make test-integration-hdfs This starts database, broker & worker containers, and also HDFS container. Then it runs only HDFS-related integration tests. +DB/filesystem addresses and credentials are stored in ``.env.local`` file. To run full test suite: @@ -219,7 +220,7 @@ And then start all necessary services: Then open http://localhost:8000/docs -Settings are stored in ``.env.docker`` file. +Settings are stored in ``config.yml`` file. Build documentation ~~~~~~~~~~~~~~~~~~~ diff --git a/Makefile b/Makefile index b5d04da8..409449ed 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ #!make -include .env.local include .env.local.test VERSION = develop diff --git a/config.docker.yml b/config.docker.yml new file mode 100644 index 00000000..702a0c37 --- /dev/null +++ b/config.docker.yml @@ -0,0 +1,68 @@ +database: + url: postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster + +broker: + url: amqp://guest:guest@rabbitmq:5672 + +encryption: + # Encrypt / Decrypt credentials data using this Fernet key. + # !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!! + secret_key: UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= + + +auth: + # Dummy Auth + provider: syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider + access_token: + secret_key: generate_another_random_string + + # Keycloak Auth + # provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider + # server_url: http://keycloak:8080 + # realm_name: manually_created + # client_id: manually_created + # client_secret: generated_by_keycloak + # redirect_uri: http://localhost:3000/auth/callback + # scope: email + # verify_ssl: False + + +ui: + api_browser_url: http://localhost:8000 + auth_provider: dummyAuth + # auth_provider: keycloakAuth + + +server: + debug: true # !!! NEVER USE ON PRODUCTION !!! + + session: + secret_key: generate_some_random_string + max_age: 86400 + + cors: + enabled: true + + allow_origins: [http://localhost:3000] + allow_credentials: true + allow_methods: ['*'] + allow_headers: ['*'] + expose_headers: [X-Request-ID, Location, Access-Control-Allow-Credentials] + + + +scheduler: + transfer_fetching_timeout_seconds: 200 + + +worker: + log_url_template: https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }} + + +hwm_store: + enabled: true + type: horizon + url: http://horizon:8000 + namespace: syncmaster_namespace + user: admin + password: 123UsedForTestOnly@! diff --git a/config.yml b/config.yml new file mode 100644 index 00000000..c20267d6 --- /dev/null +++ b/config.yml @@ -0,0 +1,71 @@ +database: + url: postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster + +broker: + url: amqp://guest:guest@localhost:5672 + +encryption: + # Encrypt / Decrypt credentials data using this Fernet key. + # !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!! + secret_key: UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= + + +auth: + # Dummy Auth + provider: syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider + access_token: + secret_key: generate_another_random_string + + # Keycloak Auth + # provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider + # server_url: http://localhost:8080 + # realm_name: manually_created + # client_id: manually_created + # client_secret: generated_by_keycloak + # redirect_uri: http://localhost:3000/auth/callback + # scope: email + # verify_ssl: False + + +ui: + api_browser_url: http://localhost:8000 + auth_provider: dummyAuth + # auth_provider: keycloakAuth + + +server: + debug: true # !!! NEVER USE ON PRODUCTION !!! + + session: + secret_key: generate_some_random_string + max_age: 86400 + + cors: + enabled: true + allow_origins: [http://localhost:3000] + allow_credentials: true + allow_methods: ['*'] + allow_headers: ['*'] + expose_headers: [X-Request-ID, Location, Access-Control-Allow-Credentials] + + + +scheduler: + transfer_fetching_timeout_seconds: 200 + + +worker: + log_url_template: https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }} + + +hwm_store: + enabled: true + type: horizon + url: http://localhost:8020 + namespace: syncmaster_namespace + user: admin + password: 123UsedForTestOnly@! + + +superusers: + - admin diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 51b4ebac..d2befac1 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -42,10 +42,10 @@ services: context: . target: test volumes: + - ./config.docker.yml:/app/config.yml - ./syncmaster:/app/syncmaster - ./tests:/app/tests entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head] - env_file: .env.docker depends_on: db: condition: service_healthy @@ -57,10 +57,10 @@ services: dockerfile: docker/Dockerfile.server context: . target: test - env_file: .env.docker ports: - 8000:8000 volumes: + - ./config.docker.yml:/app/config.yml - ./syncmaster:/app/syncmaster - ./docs/_static:/app/docs/_static - ./reports:/app/reports @@ -82,8 +82,8 @@ services: dockerfile: docker/Dockerfile.scheduler context: . target: test - env_file: .env.docker volumes: + - ./config.docker.yml:/app/config.yml - ./syncmaster:/app/syncmaster - ./tests:/app/tests - ./reports:/app/reports @@ -106,13 +106,12 @@ services: target: test command: --loglevel=info -Q 123-test_queue entrypoint: [coverage, run, -m, celery, -A, tests.test_integration.celery_test, worker, --max-tasks-per-child=1] - env_file: - - .env.docker - - .env.docker.test + env_file: .env.docker.test environment: # CI runs tests in the worker container, so we need to turn off interaction with static files for it - SYNCMASTER__SERVER__STATIC_FILES__ENABLED=false volumes: + - ./config.docker.yml:/app/config.yml - ./syncmaster:/app/syncmaster - ./reports:/app/reports - ./tests:/app/tests diff --git a/docker-compose.yml b/docker-compose.yml index 35065899..c9ffb2c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,8 @@ services: context: . target: prod entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head] - env_file: .env.docker + volumes: + - ./config.docker.yml:/app/config.yml depends_on: db: condition: service_healthy @@ -54,15 +55,13 @@ services: ports: - 8000:8000 environment: - # list here usernames which should be assigned SUPERUSER role on application start - SYNCMASTER__ENTRYPOINT__SUPERUSERS: admin # PROMETHEUS_MULTIPROC_DIR is required for multiple workers, see: # https://prometheus.github.io/client_python/multiprocess/ PROMETHEUS_MULTIPROC_DIR: /tmp/prometheus-metrics # tmpfs dir is cleaned up each container restart tmpfs: + - ./config.docker.yml:/app/config.yml - /tmp/prometheus-metrics:mode=1777 - env_file: .env.docker depends_on: db: condition: service_healthy @@ -82,8 +81,9 @@ services: dockerfile: docker/Dockerfile.worker context: . target: prod - env_file: .env.docker command: --loglevel=info -Q 123-test_queue # Queue.slug + volumes: + - ./config.docker.yml:/app/config.yml depends_on: db: condition: service_healthy @@ -102,7 +102,8 @@ services: dockerfile: docker/Dockerfile.scheduler context: . target: prod - env_file: .env.docker + volumes: + - ./config.docker.yml:/app/config.yml depends_on: db: condition: service_healthy @@ -117,7 +118,8 @@ services: frontend: image: mtsrus/syncmaster-ui:${VERSION:-develop} restart: unless-stopped - env_file: .env.docker + volumes: + - ./config.docker.yml:/app/config.yml ports: - 3000:3000 depends_on: diff --git a/docker/entrypoint_server.sh b/docker/entrypoint_server.sh index 6033dc12..d0b83281 100755 --- a/docker/entrypoint_server.sh +++ b/docker/entrypoint_server.sh @@ -1,11 +1,8 @@ #!/usr/bin/env bash set -e -if [[ "x${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" != "x" ]]; then - superusers=$(echo "${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" | tr "," " ") - python -m syncmaster.server.scripts.manage_superusers add ${superusers} - python -m syncmaster.server.scripts.manage_superusers list -fi +python -m syncmaster.server.scripts.manage_superusers add +python -m syncmaster.server.scripts.manage_superusers list # exec is required to forward all signals to the main process exec python -m syncmaster.server --host 0.0.0.0 --port 8000 "$@" diff --git a/docs/changelog/next_release/289.breaking.1.rst b/docs/changelog/next_release/289.breaking.1.rst new file mode 100644 index 00000000..97a2c1e6 --- /dev/null +++ b/docs/changelog/next_release/289.breaking.1.rst @@ -0,0 +1,2 @@ +Introduce ``config.yml`` file which is used to store settings of all components (server, scheduler, worker, frontend). +This is the main way to configure application now. It's still possible to use environment variables, but it is not recommended for security reasons. diff --git a/docs/changelog/next_release/289.breaking.2.rst b/docs/changelog/next_release/289.breaking.2.rst new file mode 100644 index 00000000..b3c16d13 --- /dev/null +++ b/docs/changelog/next_release/289.breaking.2.rst @@ -0,0 +1 @@ +Logging format is configured explicitly via ``config.yml`` instead of having a set of preset configuration files. diff --git a/docs/changelog/next_release/289.breaking.3.rst b/docs/changelog/next_release/289.breaking.3.rst new file mode 100644 index 00000000..98403421 --- /dev/null +++ b/docs/changelog/next_release/289.breaking.3.rst @@ -0,0 +1 @@ +Environment variable ``SYNCMASTER__ENTRYPOINT__SUPERUSERS`` is renamed to ``SYNCMASTER__SUPERUSERS``. diff --git a/docs/conf.py b/docs/conf.py index f4a73c72..cac9cfa6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,7 +36,7 @@ # The short X.Y version. # this value is updated automatically by `poetry version ...` and poetry-bumpversion plugin -ver = Version.parse("0.2.6") +ver = Version.parse("0.3.0") version = ver.base_version # The full version, including alpha/beta/rc tags. release = ver.public diff --git a/docs/reference/broker/index.rst b/docs/reference/broker/index.rst index b90f990f..60669608 100644 --- a/docs/reference/broker/index.rst +++ b/docs/reference/broker/index.rst @@ -30,17 +30,17 @@ With Docker ``docker-compose`` will download RabbitMQ image, create container and volume, and then start container. Image entrypoint will create database if volume is empty. - Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` - .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 33-45,144 + :emphasize-lines: 34-46,134 + + Options can be set via ``config.yml`` file: - .. dropdown:: ``.env.docker`` + .. dropdown:: ``config.yml`` - .. literalinclude:: ../../../.env.docker - :emphasize-lines: 15-16 + .. literalinclude:: ../../../config.yml + :emphasize-lines: 4-5 Without Docker ^^^^^^^^^^^^^^ diff --git a/docs/reference/database/index.rst b/docs/reference/database/index.rst index 8c39da13..5776058d 100644 --- a/docs/reference/database/index.rst +++ b/docs/reference/database/index.rst @@ -43,17 +43,17 @@ With Docker After that, one-off container with migrations script will run. - Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` - .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 1-31,142 + :emphasize-lines: 1-32,133 + + Options can be set via ``config.yml`` file: - .. dropdown:: ``.env.docker`` + .. dropdown:: ``config.yml`` - .. literalinclude:: ../../../.env.docker - :emphasize-lines: 8-9 + .. literalinclude:: ../../../config.yml + :emphasize-lines: 1-2 Without Docker ~~~~~~~~~~~~~~ @@ -73,18 +73,19 @@ Without Docker $ pip install syncmaster[postgres] -* Configure :ref:`Database connection ` using environment variables, e.g. by creating ``.env`` file: +* Configure :ref:`Database connection ` by creating config file: - .. code-block:: console - :caption: /some/.env + .. code-block:: yaml + :caption: /some/config.yml - $ export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster + database: + url: postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster - And then read values from this file: + File should be created in current directory. If not, you can override its location via environment variable: .. code:: console - $ source /some/.env + $ export SYNCMASTER_CONFIG_FILE=/some/config.yml * Run migrations: diff --git a/docs/reference/frontend/configuration.rst b/docs/reference/frontend/configuration.rst index 1b153347..d5bb861b 100644 --- a/docs/reference/frontend/configuration.rst +++ b/docs/reference/frontend/configuration.rst @@ -6,11 +6,13 @@ Frontend configuration API URL ------- -SyncMaster UI requires REST API to be accessible from browser. API url is set up using environment variable: +SyncMaster UI requires REST API to be accessible from browser. API url is set up using config file: -.. code:: bash +.. code-block:: yaml + :caption: config.yml - SYNCMASTER__UI__API_BROWSER_URL=http://localhost:8000 + ui: + api_browser_url: http://localhost:8000 If both REST API and frontend are served on the same domain (e.g. through Nginx reverse proxy), for example: @@ -19,16 +21,21 @@ If both REST API and frontend are served on the same domain (e.g. through Nginx Then you can use relative path: -.. code:: bash +.. code-block:: yaml + :caption: config.yml - SYNCMASTER__UI__API_BROWSER_URL=/api + ui: + api_browser_url: /api Auth provider ------------- By default, SyncMaster UI shows login page with username & password fields, designed for :ref:`server-auth-dummy`. -To show a login page for :ref:`keycloak-auth-provider`, you should pass this environment variable to frontend container: +To show a login page for :ref:`keycloak-auth-provider`, you should set config option: -.. code:: bash +.. code-block:: yaml + :caption: config.yml - SYNCMASTER__UI__AUTH_PROVIDER=keycloakAuthProvider + ui: + api_browser_url: keycloakAuthProvider + # api_browser_url: dummyAuth diff --git a/docs/reference/frontend/index.rst b/docs/reference/frontend/index.rst index e9226c51..424c334a 100644 --- a/docs/reference/frontend/index.rst +++ b/docs/reference/frontend/index.rst @@ -23,17 +23,17 @@ With Docker ``docker-compose`` will download SyncMaster UI image, create containers, and then start them. - Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` - .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 123-140 + :emphasize-lines: 118-130 + + Options can be set via ``config.yml`` file: - .. dropdown:: ``.env.docker`` + .. dropdown:: ``config.yml`` - .. literalinclude:: ../../../.env.docker - :emphasize-lines: 49-50 + .. literalinclude:: ../../../config.yml + :emphasize-lines: 30-33 * After frontend is started and ready, open http://localhost:3000. diff --git a/docs/reference/scheduler/configuration/logging.rst b/docs/reference/scheduler/configuration/logging.rst index 271bf571..eb841752 100644 --- a/docs/reference/scheduler/configuration/logging.rst +++ b/docs/reference/scheduler/configuration/logging.rst @@ -4,4 +4,4 @@ Logging settings ================ -.. autopydantic_model:: syncmaster.settings.log.LoggingSettings +.. autopydantic_model:: syncmaster.settings.logging.LoggingSettings diff --git a/docs/reference/scheduler/index.rst b/docs/reference/scheduler/index.rst index ffdf0777..41737b91 100644 --- a/docs/reference/scheduler/index.rst +++ b/docs/reference/scheduler/index.rst @@ -24,17 +24,17 @@ With docker ``docker-compose`` will download all necessary images, create containers, and then start the scheduler. - Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` - .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 104-121 + :emphasize-lines: 98-116 + + Options can be set via ``config.yml`` file: - .. dropdown:: ``.env.docker`` + .. dropdown:: ``config.yml`` - .. literalinclude:: ../../../.env.docker - :emphasize-lines: 8-16,37-38 + .. literalinclude:: ../../../config.yml + :emphasize-lines: 1-5,53-54 Without docker ^^^^^^^^^^^^^^ diff --git a/docs/reference/scheduler/install.rst b/docs/reference/scheduler/install.rst deleted file mode 100644 index ed7e578f..00000000 --- a/docs/reference/scheduler/install.rst +++ /dev/null @@ -1,58 +0,0 @@ -.. _server-install: - -Install & run scheduler -======================= - -With docker ------------ - -Installation process -~~~~~~~~~~~~~~~~~~~~ - -* Install `Docker `_ -* Install `docker-compose `_ - -* Run the following command: - - .. code:: console - - $ docker compose --profile scheduler up -d --wait - - ``docker-compose`` will download all necessary images, create containers, and then start the server. - - Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` - -.. dropdown:: ``docker-compose.yml`` - - .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 103-120 - -.. dropdown:: ``.env.docker`` - - .. literalinclude:: ../../../.env.docker - :emphasize-lines: 4-16,38-39 - - -Without docker --------------- - -* Install Python 3.11 or above -* Setup :ref:`database`, run migrations and create partitions -* Create virtual environment - - .. code-block:: console - - $ python -m venv /some/.venv - $ source /some/.venv/activate - -* Install ``syncmaster`` package with following *extra* dependencies: - - .. code-block:: console - - $ pip install syncmaster[scheduler] - -* Run scheduler process - - .. code-block:: console - - $ python -m syncmaster.scheduler diff --git a/docs/reference/server/auth/keycloak/local_installation.rst b/docs/reference/server/auth/keycloak/local_installation.rst index d62e800d..722e040d 100644 --- a/docs/reference/server/auth/keycloak/local_installation.rst +++ b/docs/reference/server/auth/keycloak/local_installation.rst @@ -9,61 +9,62 @@ You can test Keycloak auth locally with docker compose: $ docker compose -f docker-compose.test.yml up keycloak -d -Authorize in keycloak +Go to Keycloak Admin ~~~~~~~~~~~~~~~~~~~~~ -At first, you have to go to `http://localhost:8080/admin `_ and login via login: `admin`, password: `admin` (by default) to create realms. +Go to `Keycloak admin page `_ and sign in using login: ``admin``, password: ``admin`` (by default): .. image:: images/keycloak-login.png :width: 400px :align: center -Create new realm +Create new Realm ~~~~~~~~~~~~~~~~ +Create new Realm with some name: + .. image:: images/keycloak-new-realm.png :width: 400px :align: center - -Create new realm name -~~~~~~~~~~~~~~~~~~~~~ - -Pass realm name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME` environment variable: - -.. code-block:: console - - $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm # as on screen below - .. image:: images/keycloak-new-realm_name.png :width: 400px :align: center +Save realm name to config file: + +.. code-block:: yaml + :caption: config.yml + + auth: + keycloak: + realm_name: fastapi_realm + # ... -Create new client +Create new Client ~~~~~~~~~~~~~~~~~ +In created realm create new client and save its name to config: + .. image:: images/keycloak-new-client.png :width: 400px :align: center - -Create new client name -~~~~~~~~~~~~~~~~~~~~~~ - -In created realm pass client name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID` environment variable: - -.. code-block:: console - - $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client # as on screen below - .. image:: images/keycloak-new-client_name.png :width: 400px :align: center +.. code-block:: yaml + :caption: config.yml + + auth: + keycloak: + client_id: fastapi_client + # ... -Set ``client_authentication`` **ON** to receive client_secret + +Set ``client_authentication=ON`` to generate client_secret ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. image:: images/keycloak-client-authentication.png @@ -73,49 +74,62 @@ Set ``client_authentication`` **ON** to receive client_secret Configure Redirect URI ~~~~~~~~~~~~~~~~~~~~~~ -To configure the redirect URI where the browser will redirect to exchange the code provided from Keycloak for an access token, set the `SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI` environment variable. The default value for local development is `http://localhost:3000/auth/callback`. - -.. code-block:: console - - $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:3000/auth/callback - -Configure the client redirect URI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Ensure that this URI is also configured as a valid redirect URI in your Keycloak client settings. This allows the browser to redirect to your application after the user successfully authenticates with Keycloak. +Set URI to redirect from Keycloak login page for exchanging the code for an access token: .. image:: images/keycloak-client-redirect_uri.png :width: 400px :align: center -Configure the client secret -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: yaml + :caption: config.yml -Now go to **Credentials** tab and add the client secret to the `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET` environment variable: + auth: + keycloak: + # set here SyncMaster hostname/domain + redirect_uri: http://localhost:3000/auth/callback + # ... -.. code-block:: console +Configure the client secret +~~~~~~~~~~~~~~~~~~~~~~~~~~~ - $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez # as on screen below +Now go to **Credentials** tab and generate a client secret: .. image:: images/keycloak-client-secret.png :width: 400px :align: center -Now you can use create users in this realms, check `keycloak documentation `_ on how to manage users creation. - -ENVIRONMENT VARIABLES -~~~~~~~~~~~~~~~~~~~~~ - -After this you can user `KeycloakAuthProvider` in your application with provided environment variables: - - -.. code-block:: console - - $ export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 - $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:3000/auth/callback - $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm - $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client - $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez - $ export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email - $ export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False - $ export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider +.. code-block:: yaml + :caption: config.yml + + auth: + keycloak: + client_id: fastapi_client + client_secret: 6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez + +Now you can use create users in this realm, check `Keycloak documentation `_ on how to manage users creation. + +Final configuration +~~~~~~~~~~~~~~~~~~~ + +After this you can use ``KeycloakAuthProvider`` in your application: + +.. code-block:: yaml + :caption: config.yml + + auth: + provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider + keycloak: + # Keycloak URL accessible from both SyncMaster server and from browser + server_url: http://keycloak:8080 + # set here SyncMaster hostname/domain + redirect_uri: http://localhost:3000/auth/callback + realm_name: fastapi_realm + client_id: fastapi_client + client_secret: 6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez + scope: email + verify_ssl: false + + ui: + auth_provider: keycloakAuth + # set here SyncMaster hostname/domain + api_browser_url: http://localhost:8000 diff --git a/docs/reference/server/configuration/debug.rst b/docs/reference/server/configuration/debug.rst index bc85e4a3..393f2b48 100644 --- a/docs/reference/server/configuration/debug.rst +++ b/docs/reference/server/configuration/debug.rst @@ -9,12 +9,16 @@ Return debug info in REST API responses By default, server does not add error details to response bodies, to avoid exposing instance-specific information to end users. -You can change this by setting: +With debug enabled: + +.. code-block:: yaml + :caption: config.yml + + server: + debug: true .. code-block:: console - $ export SYNCMASTER__SERVER__DEBUG=False - $ # start REST API server $ curl -XPOST http://localhost:8000/failing/endpoint ... { "error": { @@ -24,10 +28,16 @@ You can change this by setting: }, } +With debug disabled: + +.. code-block:: yaml + :caption: config.yml + + server: + debug: false + .. code-block:: console - $ export SYNCMASTER__SERVER__DEBUG=True - $ # start REST API server $ curl -XPOST http://localhost:8000/failing/endpoint ... Traceback (most recent call last): File ".../uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi @@ -57,12 +67,19 @@ This is done by ``request_id`` middleware, which is enabled by default and can c Print request ID to backend logs --------------------------------- -This is done by adding a specific filter to logging handler: +This is done by adding a specific filter to logging settings: .. dropdown:: ``logging.yml`` - .. literalinclude:: ../../../../syncmaster/settings/log/plain.yml - :emphasize-lines: 6-12,17-18,25 + .. code-block:: yaml + :caption: config.yml + + logging: + filters: + correlation_id: + class: asgi_correlation_id.CorrelationIdFilter + uuid_length: 32 + default_value: '-' Resulting logs look like: diff --git a/docs/reference/server/configuration/logging.rst b/docs/reference/server/configuration/logging.rst index e3ffd7f9..5940bae9 100644 --- a/docs/reference/server/configuration/logging.rst +++ b/docs/reference/server/configuration/logging.rst @@ -4,4 +4,4 @@ Logging settings ================ -.. autopydantic_model:: syncmaster.settings.log.LoggingSettings +.. autopydantic_model:: syncmaster.settings.logging.LoggingSettings diff --git a/docs/reference/server/index.rst b/docs/reference/server/index.rst index 57316cdd..0fa157bc 100644 --- a/docs/reference/server/index.rst +++ b/docs/reference/server/index.rst @@ -23,17 +23,17 @@ With docker ``docker-compose`` will download all necessary images, create containers, and then start the server. - Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` - .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 47-82 + :emphasize-lines: 48-75 + + Options can be set via ``config.yml`` file: - .. dropdown:: ``.env.docker`` + .. dropdown:: ``config.yml`` - .. literalinclude:: ../../../.env.docker - :emphasize-lines: 4-35 + .. literalinclude:: ../../../config.yml + :emphasize-lines: 7-27, 36-49, 70-71 * After server is started and ready, open http://localhost:8000/docs. diff --git a/docs/reference/server/manage_superusers_cli.rst b/docs/reference/server/manage_superusers_cli.rst index 09f5d38d..070d3de2 100644 --- a/docs/reference/server/manage_superusers_cli.rst +++ b/docs/reference/server/manage_superusers_cli.rst @@ -5,10 +5,25 @@ CLI for managing superusers There are two ways to manage users: -* automatic: +* automatic - Set ``SYNCMASTER__ENTRYPOINT__SUPERUSERS=user1,user2``, and :ref:`server` Docker container entrypoint - will automatically set ``is_superuser=True`` flag for them, and reset for other users in database. + :ref:`server` Docker container entrypoint will automatically create users with ``is_superuser=True`` in database + during startup. + + Usernames can be passed via config file: + + .. code-block:: yaml + :caption: config.yml + + superusers: + - user1 + - user2 + + Or via enviroment variable: + + .. code-block:: bash + + export 'SYNCMASTER__SUPERUSERS=["user1", "user2"]' * manual via CLI: diff --git a/docs/reference/worker/configuration/logging.rst b/docs/reference/worker/configuration/logging.rst index 4f647075..7176685e 100644 --- a/docs/reference/worker/configuration/logging.rst +++ b/docs/reference/worker/configuration/logging.rst @@ -4,4 +4,4 @@ Logging settings ================ -.. autopydantic_model:: syncmaster.settings.log.LoggingSettings +.. autopydantic_model:: syncmaster.settings.logging.LoggingSettings diff --git a/docs/reference/worker/create_spark_session.rst b/docs/reference/worker/create_spark_session.rst index 76891f84..c4d47dde 100644 --- a/docs/reference/worker/create_spark_session.rst +++ b/docs/reference/worker/create_spark_session.rst @@ -8,9 +8,11 @@ By default, SparkSession is created with ``master=local``, all required .jar pac It is possible to alter SparkSession config by providing custom function: -.. code-block:: bash +.. code-block:: yaml + :caption: config.yml - SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=my_worker.spark.create_custom_spark_session + worker: + create_spark_session_function: my_worker.spark.create_custom_spark_session Here is a function example: diff --git a/docs/reference/worker/index.rst b/docs/reference/worker/index.rst index 13980564..40d8f8d0 100644 --- a/docs/reference/worker/index.rst +++ b/docs/reference/worker/index.rst @@ -33,17 +33,17 @@ With docker ``docker-compose`` will download all necessary images, create containers, and then start the worker. - Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` - .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 84-102 + :emphasize-lines: 77-96 + + Options can be set via ``config.yml`` file: - .. dropdown:: ``.env.docker`` + .. dropdown:: ``config.yml`` - .. literalinclude:: ../../../.env.docker - :emphasize-lines: 8-16,40-47 + .. literalinclude:: ../../../config.yml + :emphasize-lines: 1-10,57-67 Without docker ^^^^^^^^^^^^^^ diff --git a/docs/reference/worker/log_url.rst b/docs/reference/worker/log_url.rst index 2906c924..5ceceefe 100644 --- a/docs/reference/worker/log_url.rst +++ b/docs/reference/worker/log_url.rst @@ -9,8 +9,10 @@ This log URL might point to an Elastic instance or another logging tool such as The log URL is generated based on a template configured in the configuration. The configuration parameter is: -.. code-block:: bash +.. code-block:: yaml + :caption: config.yml - SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} + worker: + log_url_template: https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} In this example, run logs can be retrieved by either its correlation id ``x-request-id`` in http headers, or by ``Run.Id`` field value. diff --git a/poetry.lock b/poetry.lock index 0a697c1d..cfdc3150 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2924,6 +2924,22 @@ gcp-secret-manager = ["google-cloud-secret-manager (>=2.23.1)"] toml = ["tomli (>=2.0.1)"] yaml = ["pyyaml (>=6.0.1)"] +[[package]] +name = "pydantic-settings-logging" +version = "0.1.1" +description = "Pydantic models for Python logging configuration" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "pydantic_settings_logging-0.1.1-py3-none-any.whl", hash = "sha256:08a961882a6ef6bde4fc4f7ec121aa29ee46cbd2f013974702c9a57f84e6204b"}, + {file = "pydantic_settings_logging-0.1.1.tar.gz", hash = "sha256:745879f91fe742b359765aa58bb42ea998985a04574391fc202fbd6c783e1715"}, +] + +[package.dependencies] +pydantic = ">=2.0" +pydantic-settings = ">=2.0" + [[package]] name = "pyflakes" version = "3.4.0" @@ -4347,4 +4363,4 @@ worker = ["horizon-hwm-store", "jinja2", "onetl", "psycopg2-binary", "pyspark"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "ece82f25affbf7251f2b388f2e98e62022520748fcaffe5f12bc42100797c205" +content-hash = "29ce2388721292623945956cb2c28009b0377532ef6c7d07135ce45432a264a2" diff --git a/pyproject.toml b/pyproject.toml index 178964f2..5de66be4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "data-syncmaster" -version = "0.2.6" +version = "0.3.0" license = "Apache-2.0" description = "Syncmaster REST API + Worker" authors = ["DataOps.ETL "] @@ -46,6 +46,7 @@ exclude = [ python = "^3.11" pydantic = "^2.12.4" pydantic-settings = "^2.8.1" +pydantic-settings-logging = "^0.1.1" sqlalchemy = "^2.0.40" sqlalchemy-utils = "^0.42.0" pyyaml = "^6.0.3" diff --git a/syncmaster/__init__.py b/syncmaster/__init__.py index 3bee545e..30b451b8 100644 --- a/syncmaster/__init__.py +++ b/syncmaster/__init__.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 -_raw_version = "0.2.6" +_raw_version = "0.3.0" # version always contain only release number like 0.0.1 __version__ = ".".join(_raw_version.split(".")[:3]) # noqa: WPS410 diff --git a/syncmaster/db/migrations/env.py b/syncmaster/db/migrations/env.py index be7d2140..555befd6 100644 --- a/syncmaster/db/migrations/env.py +++ b/syncmaster/db/migrations/env.py @@ -7,23 +7,34 @@ from alembic import context from alembic.script import ScriptDirectory from celery.backends.database.session import ResultModelBase +from pydantic import Field from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config from syncmaster.db.models import Base -from syncmaster.server.settings import ServerAppSettings as Settings +from syncmaster.server.settings import ( + DEFAULT_LOGGING_SETTINGS, + BaseSettings, + DatabaseSettings, + LoggingSettings, +) config = context.config +class MigrationAppSettings(BaseSettings): + database: DatabaseSettings = Field(default_factory=DatabaseSettings, description="Database settings") + logging: LoggingSettings = Field(default=DEFAULT_LOGGING_SETTINGS, description="Logging settings") + + if config.config_file_name is not None: fileConfig(config.config_file_name) if not config.get_main_option("sqlalchemy.url"): # read application settings only if sqlalchemy.url is not being passed via cli arguments # TODO: remove settings object creating during import - config.set_main_option("sqlalchemy.url", Settings().database.url) + config.set_main_option("sqlalchemy.url", MigrationAppSettings().database.url) target_metadata = ( Base.metadata, diff --git a/syncmaster/scheduler/__main__.py b/syncmaster/scheduler/__main__.py index 45a9995e..5c097eee 100755 --- a/syncmaster/scheduler/__main__.py +++ b/syncmaster/scheduler/__main__.py @@ -7,14 +7,14 @@ from syncmaster.scheduler.settings import SchedulerAppSettings as Settings from syncmaster.scheduler.transfer_fetcher import TransferFetcher from syncmaster.scheduler.transfer_job_manager import TransferJobManager -from syncmaster.settings.log import setup_logging +from syncmaster.settings.logging import setup_logging logger = logging.getLogger(__name__) async def main(): settings = Settings() - setup_logging(settings.logging.get_log_config_path()) + setup_logging(settings.logging) transfer_fetcher = TransferFetcher(settings) transfer_job_manager = TransferJobManager(settings) transfer_job_manager.scheduler.start() diff --git a/syncmaster/scheduler/settings/__init__.py b/syncmaster/scheduler/settings/__init__.py index 4a8e864a..cacf08db 100644 --- a/syncmaster/scheduler/settings/__init__.py +++ b/syncmaster/scheduler/settings/__init__.py @@ -1,9 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import BaseModel, Field from syncmaster.settings import ( + DEFAULT_LOGGING_SETTINGS, + BaseSettings, CredentialsEncryptionSettings, DatabaseSettings, LoggingSettings, @@ -11,15 +12,18 @@ ) -class SchedulerSettings(BaseSettings): - """Celery scheduler settings. +class SchedulerSettings(BaseModel): + """Scheduler settings. Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 + scheduler: + transfer_fetching_timeout_seconds: 200 + misfire_grace_time_seconds: 300 """ TRANSFER_FETCHING_TIMEOUT_SECONDS: int = Field( @@ -36,14 +40,11 @@ class SchedulerAppSettings(BaseSettings): """ Scheduler application settings. - This class is used to configure various settings for the scheduler application. - The settings can be defined in two ways: + The settings can be passed in several ways: - 1. By explicitly passing a settings object as an argument. - 2. By setting environment variables matching specific keys. - - All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. - Nested items are delimited with ``__``. + 1. By storing settings in a configuration file ``config.yml`` (preferred). + 2. By setting environment variables matching specific keys (``SYNCMASTER__DATABASE__URL`` == ``database.url``). + 3. By explicitly passing a settings object as an argument of application factory function. More details can be found in `Pydantic documentation `_. @@ -51,28 +52,25 @@ class SchedulerAppSettings(BaseSettings): Examples -------- - .. code-block:: bash - - # Example of setting a TRANSFER_FETCHING_TIMEOUT_SECONDS via environment variable - SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 + .. code-block:: yaml + :caption: config.yml - # Example of setting a database URL via environment variable - SYNCMASTER__DATABASE__URL=postgresql+asyncpg://user:password@localhost:5432/dbname + database: + url: postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster - # Example of setting a broker URL via environment variable - SYNCMASTER__BROKER__URL=amqp://user:password@localhost:5672/ + broker: + url: amqp://user:password@localhost:5672/ - Refer to `Pydantic documentation `_ - for more details on configuration options and environment variable usage. + logging: {} + encryption: {} + scheduler: {} """ - database: DatabaseSettings = Field(description="Database settings") - broker: RabbitMQSettings = Field(description="Broker settings") - logging: LoggingSettings = Field(default_factory=LoggingSettings, description="Logging settings") + database: DatabaseSettings = Field(default_factory=DatabaseSettings, description="Database settings") + broker: RabbitMQSettings = Field(default_factory=RabbitMQSettings, description="Broker settings") + logging: LoggingSettings = Field(default=DEFAULT_LOGGING_SETTINGS, description="Logging settings") scheduler: SchedulerSettings = Field(default_factory=SchedulerSettings, description="Scheduler-specific settings") encryption: CredentialsEncryptionSettings = Field( default_factory=CredentialsEncryptionSettings, description="Settings for encrypting credential data", ) - - model_config = SettingsConfigDict(env_prefix="SYNCMASTER__", env_nested_delimiter="__") diff --git a/syncmaster/schemas/v1/transfers/__init__.py b/syncmaster/schemas/v1/transfers/__init__.py index dec6e586..2a35ce42 100644 --- a/syncmaster/schemas/v1/transfers/__init__.py +++ b/syncmaster/schemas/v1/transfers/__init__.py @@ -200,7 +200,7 @@ class CreateTransferSchema(BaseModel): description="Incremental or archive download options", ) transformations: list[ - Annotated[TransformationSchema, Field(None, discriminator="type", description="List of transformations")] + Annotated[TransformationSchema, Field(discriminator="type", description="List of transformations")] ] = Field(default_factory=list) resources: Resources = Field( default_factory=Resources, diff --git a/syncmaster/server/__init__.py b/syncmaster/server/__init__.py index 8272332f..63dc33b8 100644 --- a/syncmaster/server/__init__.py +++ b/syncmaster/server/__init__.py @@ -20,6 +20,7 @@ from syncmaster.server.providers.auth import AuthProvider from syncmaster.server.services.unit_of_work import UnitOfWork from syncmaster.server.settings import ServerAppSettings as Settings +from syncmaster.settings.logging import setup_logging __all__ = ["get_application", "application_factory"] @@ -71,4 +72,6 @@ def application_factory(settings: Settings) -> FastAPI: def get_application() -> FastAPI: - return application_factory(settings=Settings()) + settings = Settings() + setup_logging(settings.logging) + return application_factory(settings=settings) diff --git a/syncmaster/server/middlewares/__init__.py b/syncmaster/server/middlewares/__init__.py index de392c72..e62a8d4e 100644 --- a/syncmaster/server/middlewares/__init__.py +++ b/syncmaster/server/middlewares/__init__.py @@ -12,7 +12,6 @@ from syncmaster.server.middlewares.session import apply_session_middleware from syncmaster.server.middlewares.static_files import apply_static_files from syncmaster.server.settings import ServerAppSettings as Settings -from syncmaster.settings.log import setup_logging def apply_middlewares( @@ -21,9 +20,6 @@ def apply_middlewares( ) -> FastAPI: """Add middlewares to the application.""" - if settings.logging.setup: - setup_logging(settings.logging.get_log_config_path()) - apply_cors_middleware(application, settings.server.cors) apply_monitoring_metrics_middleware(application, settings.server.monitoring) apply_request_id_middleware(application, settings.server.request_id) diff --git a/syncmaster/server/scripts/manage_superusers.py b/syncmaster/server/scripts/manage_superusers.py index a858e20a..879c70e9 100755 --- a/syncmaster/server/scripts/manage_superusers.py +++ b/syncmaster/server/scripts/manage_superusers.py @@ -8,12 +8,33 @@ import asyncio import logging +from pydantic import Field from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.future import select from syncmaster.db.models.user import User -from syncmaster.server.middlewares import setup_logging -from syncmaster.server.settings import ServerAppSettings as Settings +from syncmaster.server.settings import ( + DEFAULT_LOGGING_SETTINGS, + BaseSettings, + DatabaseSettings, + LoggingSettings, +) +from syncmaster.settings.logging import setup_logging + + +class SuperuserAppSettings(BaseSettings): + database: DatabaseSettings = Field( + default_factory=DatabaseSettings, # type: ignore[arg-type] + description="Database settings", + ) + logging: LoggingSettings = Field( + default=DEFAULT_LOGGING_SETTINGS, + description="Logging settings", + ) + superusers: list[str] = Field( + default_factory=list, + description="List of superuser usernames", + ) async def add_superusers(session: AsyncSession, usernames: list[str]) -> None: @@ -93,13 +114,15 @@ async def main(args: argparse.Namespace, session: AsyncSession) -> None: if __name__ == "__main__": - settings = Settings() - if settings.logging.setup: - setup_logging(settings.logging.get_log_config_path()) + settings = SuperuserAppSettings() + setup_logging(settings.logging) engine = create_async_engine(settings.database.url) SessionLocal = async_sessionmaker(autocommit=False, autoflush=False, bind=engine, class_=AsyncSession) parser = create_parser() args = parser.parse_args() + if not args.usernames: + args.usernames = settings.superusers + session = SessionLocal() asyncio.run(main(args, session)) diff --git a/syncmaster/server/settings/__init__.py b/syncmaster/server/settings/__init__.py index a2c7c9b8..a7010419 100644 --- a/syncmaster/server/settings/__init__.py +++ b/syncmaster/server/settings/__init__.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict from syncmaster.server.settings.auth import AuthSettings from syncmaster.server.settings.server import ServerSettings from syncmaster.settings import ( + DEFAULT_LOGGING_SETTINGS, + BaseSettings, CredentialsEncryptionSettings, DatabaseSettings, LoggingSettings, @@ -14,15 +15,13 @@ class ServerAppSettings(BaseSettings): - """Syncmaster server settings. + """Server application settings. - Server can be configured in 2 ways: + The settings can be passed in several ways: - * By explicitly passing ``settings`` object as an argument to :obj:`application_factory ` - * By setting up environment variables matching a specific key. - - All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. - Nested items are delimited with ``__``. + 1. By storing settings in a configuration file ``config.yml`` (preferred). + 2. By setting environment variables matching specific keys (``SYNCMASTER__DATABASE__URL`` == ``database.url``). + 3. By explicitly passing a settings object as an argument of application factory function. More details can be found in `Pydantic documentation `_. @@ -30,19 +29,31 @@ class ServerAppSettings(BaseSettings): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml + + database: + url: postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster - # same as settings.database.url = "postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster" - SYNCMASTER__DATABASE__URL=postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster + broker: + url: amqp://user:password@localhost:5672/ - # same as settings.server.debug = True - SYNCMASTER__SERVER__DEBUG=True + logging: {} + encryption: {} + server: {} + auth: {} """ - database: DatabaseSettings = Field(description=":ref:`Database settings `") - broker: RabbitMQSettings = Field(description=":ref:`Broker settings `") + database: DatabaseSettings = Field( + default_factory=DatabaseSettings, # type: ignore[arg-type] + description=":ref:`Database settings `", + ) + broker: RabbitMQSettings = Field( + default_factory=RabbitMQSettings, # type: ignore[arg-type] + description=":ref:`Broker settings `", + ) logging: LoggingSettings = Field( - default_factory=LoggingSettings, + default=DEFAULT_LOGGING_SETTINGS, description=":ref:`Logging settings `", ) server: ServerSettings = Field( @@ -57,5 +68,3 @@ class ServerAppSettings(BaseSettings): default_factory=CredentialsEncryptionSettings, # type: ignore[arg-type] description="Settings for encrypting credential data", ) - - model_config = SettingsConfigDict(env_prefix="SYNCMASTER__", env_nested_delimiter="__") diff --git a/syncmaster/server/settings/auth/__init__.py b/syncmaster/server/settings/auth/__init__.py index 9a1a7710..4324d06e 100644 --- a/syncmaster/server/settings/auth/__init__.py +++ b/syncmaster/server/settings/auth/__init__.py @@ -12,12 +12,14 @@ class AuthSettings(BaseModel): Examples -------- - .. code-block:: bash - - SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider - - # pass access_key.secret_key = "secret" to DummyAuthProviderSettings - SYNCMASTER__AUTH__ACCESS_KEY__SECRET_KEY=secret + .. code-block:: yaml + :caption: config.yml + + auth: + provider: syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider + # other options passed to AuthProviderSettings, e.g. DummyAuthProviderSettings + access_key: + secret_key: jwt_secret """ provider: ImportString = Field( # type: ignore[assignment] diff --git a/syncmaster/server/settings/auth/dummy.py b/syncmaster/server/settings/auth/dummy.py index 3ea5db4e..143a1d3a 100644 --- a/syncmaster/server/settings/auth/dummy.py +++ b/syncmaster/server/settings/auth/dummy.py @@ -11,10 +11,13 @@ class DummyAuthProviderSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider - SYNCMASTER__AUTH__ACCESS_KEY__SECRET_KEY=secret + auth: + provider: syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider + access_key: + secret_key: jwt_secret """ access_token: JWTSettings = Field(description="Access-token related settings") diff --git a/syncmaster/server/settings/auth/jwt.py b/syncmaster/server/settings/auth/jwt.py index c1c8ef03..db35c1b8 100644 --- a/syncmaster/server/settings/auth/jwt.py +++ b/syncmaster/server/settings/auth/jwt.py @@ -12,10 +12,13 @@ class JWTSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__AUTH__ACCESS_KEY__SECRET_KEY=somesecret - SYNCMASTER__AUTH__ACCESS_KEY__EXPIRE_SECONDS=3600 # 1 hour + auth: + access_key: + secret_key: jwt_secret + expire_seconds: 3600 # 1 hour """ secret_key: SecretStr = Field( diff --git a/syncmaster/server/settings/auth/keycloak.py b/syncmaster/server/settings/auth/keycloak.py index a27af025..af665641 100644 --- a/syncmaster/server/settings/auth/keycloak.py +++ b/syncmaster/server/settings/auth/keycloak.py @@ -7,15 +7,32 @@ class KeycloakSettings(BaseModel): server_url: str = Field(description="Keycloak server URL") client_id: str = Field(description="Keycloak client ID") - realm_name: str = Field(description="Keycloak realm name") client_secret: SecretStr = Field(description="Keycloak client secret") + realm_name: str = Field(description="Keycloak realm name") redirect_uri: str = Field(description="Redirect URI") - verify_ssl: bool = Field(True, description="Verify SSL certificates") - scope: str = Field("openid", description="Keycloak scope") + verify_ssl: bool = Field(default=True, description="Verify SSL certificates") + scope: str = Field(default="openid", description="Keycloak scope") class KeycloakAuthProviderSettings(BaseModel): - """Settings related to Keycloak interaction.""" + """Settings for KeycloakAuthProvider. + + Examples + -------- + + .. code-block:: yaml + :caption: config.yml + + auth: + provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider + server_url: http://localhost:8080/auth + client_id: my_keycloak_client + client_secret: keycloak_client_secret + realm_name: my_realm + redirect_uri: http://localhost:8000/auth/realms/my_realm/protocol/openid-connect/auth + verify_ssl: false + scope: openid + """ keycloak: KeycloakSettings = Field( description="Keycloak settings", diff --git a/syncmaster/server/settings/auth/oauth2_gateway.py b/syncmaster/server/settings/auth/oauth2_gateway.py index 6696df4b..4abe4c82 100644 --- a/syncmaster/server/settings/auth/oauth2_gateway.py +++ b/syncmaster/server/settings/auth/oauth2_gateway.py @@ -6,7 +6,24 @@ class OAuth2GatewayProviderSettings(BaseModel): - """Settings related to Keycloak interaction.""" + """Settings for OAuth2GatewayProvider. + + Examples + -------- + + .. code-block:: yaml + :caption: config.yml + + auth: + provider: syncmaster.server.providers.auth.oauth2_gateway_provider.OAuth2GatewayProvider + server_url: http://localhost:8080/auth + client_id: my_keycloak_client + client_secret: keycloak_client_secret + realm_name: my_realm + redirect_uri: http://localhost:8000/auth/realms/my_realm/protocol/openid-connect/auth + verify_ssl: false + scope: openid + """ keycloak: KeycloakSettings = Field( description="Keycloak settings", diff --git a/syncmaster/server/settings/server/__init__.py b/syncmaster/server/settings/server/__init__.py index cad414df..2068b319 100644 --- a/syncmaster/server/settings/server/__init__.py +++ b/syncmaster/server/settings/server/__init__.py @@ -19,11 +19,23 @@ class ServerSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SERVER__DEBUG=True - SYNCMASTER__SERVER__REQUEST_ID__ENABLED=True - SYNCMASTER__SERVER__MONITORING__ENABLED=True + server: + debug: true + request_id: + enabled: true + session: + secret_key: super-secret-key + cors: + enabled: true + monitoring: + enabled: true + openapi: + enabled: true + static_files: + enabled: true """ debug: bool = Field( diff --git a/syncmaster/server/settings/server/cors.py b/syncmaster/server/settings/server/cors.py index 3d376f5a..516b00c9 100644 --- a/syncmaster/server/settings/server/cors.py +++ b/syncmaster/server/settings/server/cors.py @@ -1,11 +1,7 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 - -import json -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field class CORSSettings(BaseModel): @@ -23,25 +19,31 @@ class CORSSettings(BaseModel): For development environment: - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SERVER__CORS__ENABLED=True - SYNCMASTER__SERVER__CORS__ALLOW_ORIGINS="*" - SYNCMASTER__SERVER__CORS__ALLOW_METHODS="*" - SYNCMASTER__SERVER__CORS__ALLOW_HEADERS="*" - SYNCMASTER__SERVER__CORS__EXPOSE_HEADERS=X-Request-ID,Location,Access-Control-Allow-Credentials + server: + cors: + enabled: True + allow_origins: ["*"] + allow_methods: ["*"] + allow_headers: ["*"] + expose_headers: [X-Request-ID, Location, Access-Control-Allow-Credentials] For production environment: - .. code-block:: bash - - SYNCMASTER__SERVER__CORS__ENABLED=True - SYNCMASTER__SERVER__CORS__ALLOW_ORIGINS="production.example.com" - SYNCMASTER__SERVER__CORS__ALLOW_METHODS="GET" - SYNCMASTER__SERVER__CORS__ALLOW_HEADERS="X-Request-ID,X-Request-With" - SYNCMASTER__SERVER__CORS__EXPOSE_HEADERS="X-Request-ID" - # custom option passed directly to middleware - SYNCMASTER__SERVER__CORS__MAX_AGE=600 + .. code-block:: yaml + :caption: config.yml + + server: + cors: + enabled: True + allow_origins: [production.example.com] + allow_methods: [GET] + allow_headers: [X-Request-ID, X-Request-With] + expose_headers: [X-Request-ID] + # custom option passed directly to middleware + max_age: 600 """ enabled: bool = Field(default=True, description="Set to ``True`` to enable middleware") @@ -58,13 +60,4 @@ class CORSSettings(BaseModel): ) expose_headers: list[str] = Field(default=["X-Request-ID"], description="HTTP headers exposed from server") - @field_validator("allow_origins", "allow_methods", "allow_headers", "expose_headers", mode="before") - @classmethod - def _validate_bootstrap_servers(cls, value: Any): - if not isinstance(value, str): - return value - if "[" in value: - return json.loads(value) - return [item.strip() for item in value.split(",")] - model_config = ConfigDict(extra="allow") diff --git a/syncmaster/server/settings/server/monitoring.py b/syncmaster/server/settings/server/monitoring.py index 3f5e6405..73412efb 100644 --- a/syncmaster/server/settings/server/monitoring.py +++ b/syncmaster/server/settings/server/monitoring.py @@ -18,11 +18,22 @@ class MonitoringSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SERVER__MONITORING__ENABLED=True - SYNCMASTER__SERVER__MONITORING__SKIP_PATHS=["/some/path"] - SYNCMASTER__SERVER__MONITORING__SKIP_METHODS=["OPTIONS"] + server: + monitoring: + enabled: True + labels: + instance: "production" + skip_paths: + - "/some/path" + skip_methods: + - OPTIONS + group_paths: True + filter_unhandled_paths: True + + # custom option passed directly to starlette-exporter """ enabled: bool = Field(default=True, description="Set to ``True`` to enable middleware") diff --git a/syncmaster/server/settings/server/openapi.py b/syncmaster/server/settings/server/openapi.py index 95e2d0bd..cd7375d1 100644 --- a/syncmaster/server/settings/server/openapi.py +++ b/syncmaster/server/settings/server/openapi.py @@ -14,11 +14,16 @@ class SwaggerSettings(BaseModel): Examples -------- - .. code-block:: bash - - SYNCMASTER__SERVER__OPENAPI__SWAGGER__ENABLED=True - SYNCMASTER__SERVER__OPENAPI__SWAGGER__JS_URL=/static/swagger/swagger-ui-bundle.js - SYNCMASTER__SERVER__OPENAPI__SWAGGER__CSS_URL=/static/swagger/swagger-ui.css + .. code-block:: yaml + :caption: config.yml + + server: + openapi: + swagger: + enabled: True + js_url: /static/swagger/swagger-ui-bundle.js + css_url: /static/swagger/swagger-ui.css + extra_parameters: {} """ enabled: bool = Field(default=True, description="Set to ``True`` to enable Swagger UI endpoint") @@ -49,10 +54,14 @@ class RedocSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SERVER__OPENAPI__REDOC__ENABLED=True - SYNCMASTER__SERVER__OPENAPI__REDOC__JS_URL=/static/redoc/redoc.standalone.js + server: + openapi: + redoc: + enabled: True + js_url: /static/redoc/redoc.standalone.js """ enabled: bool = Field(default=True, description="Set to ``True`` to enable Redoc UI endpoint") @@ -71,12 +80,16 @@ class LogoSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SERVER__OPENAPI__LOGO__URL=/static/logo.svg - SYNCMASTER__SERVER__OPENAPI__LOGO__BACKGROUND_COLOR=ffffff - SYNCMASTER__SERVER__OPENAPI__LOGO__ALT_TEXT=Syncmaster logo - SYNCMASTER__SERVER__OPENAPI__LOGO__HREF=http://mycompany.domain.com + server: + openapi: + logo: + url: /static/logo.svg + background_color: ffffff + alt_text: Syncmaster logo + href: http://mycompany.domain.com """ url: str = Field( @@ -103,9 +116,13 @@ class FaviconSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SERVER__OPENAPI__FAVICON__URL=/static/icon.svg + server: + openapi: + favicon: + url: /static/icon.svg """ url: str = Field( @@ -122,11 +139,20 @@ class OpenAPISettings(BaseModel): Examples -------- - .. code-block:: bash - - SYNCMASTER__SERVER__OPENAPI__ENABLED=True - SYNCMASTER__SERVER__OPENAPI__SWAGGER__ENABLED=True - SYNCMASTER__SERVER__OPENAPI__REDOC__ENABLED=True + .. code-block:: yaml + :caption: config.yml + + server: + openapi: + enabled: True + swagger: + enabled: True + redoc: + enabled: True + logo: + url: /static/logo.svg + favicon: + url: /static/icon.svg """ enabled: bool = Field(default=True, description="Set to ``True`` to enable OpenAPI.json endpoint") diff --git a/syncmaster/server/settings/server/request_id.py b/syncmaster/server/settings/server/request_id.py index e373d9b6..c03f2774 100644 --- a/syncmaster/server/settings/server/request_id.py +++ b/syncmaster/server/settings/server/request_id.py @@ -14,10 +14,14 @@ class RequestIDSettings(BaseModel): Examples -------- - .. code-block:: bash - - SYNCMASTER__SERVER__REQUEST_ID__ENABLED=True - SYNCMASTER__SERVER__REQUEST_ID__UPDATE_REQUEST_HEADER=True + .. code-block:: yaml + :caption: config.yml + + server: + request_id: + enabled: True + header_name: X-Request-ID + update_request_header: True """ enabled: bool = Field(default=True, description="Set to ``True`` to enable middleware") diff --git a/syncmaster/server/settings/server/session.py b/syncmaster/server/settings/server/session.py index 01010c0b..e0724f32 100644 --- a/syncmaster/server/settings/server/session.py +++ b/syncmaster/server/settings/server/session.py @@ -22,21 +22,31 @@ class SessionSettings(BaseModel): For development environment: - .. code-block:: bash - - SYNCMASTER__SERVER__SESSION__SECRET_KEY=secret - SYNCMASTER__SERVER__SESSION__SESSION_COOKIE=custom_cookie_name - SYNCMASTER__SERVER__SESSION__MAX_AGE=None # cookie will last as long as the browser session - SYNCMASTER__SERVER__SESSION__SAME_SITE=strict - SYNCMASTER__SERVER__SESSION__HTTPS_ONLY=True - SYNCMASTER__SERVER__SESSION__DOMAIN=example.com + .. code-block:: yaml + :caption: config.yml + + server: + session: + secret_key: cookie_secret + session_cookie: custom_cookie_name + max_age: null + same_site: lax + https_only: True + domain: localhost For production environment: - .. code-block:: bash - - SYNCMASTER__SERVER__SESSION__SECRET_KEY=secret - SYNCMASTER__SERVER__SESSION__HTTPS_ONLY=True + .. code-block:: yaml + :caption: config.yml + + server: + session: + secret_key: cookie_secret + session_cookie: custom_cookie_name + max_age: 3600 + same_site: strict + https_only: True + domain: example.com """ diff --git a/syncmaster/server/settings/server/static_files.py b/syncmaster/server/settings/server/static_files.py index ca6bddcb..725c48f3 100644 --- a/syncmaster/server/settings/server/static_files.py +++ b/syncmaster/server/settings/server/static_files.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from pathlib import Path -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, ValidationInfo, field_validator class StaticFilesSettings(BaseModel): @@ -13,10 +13,13 @@ class StaticFilesSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__SERVER__STATIC_FILES__ENABLED=True - SYNCMASTER__SERVER__STATIC_FILES__DIRECTORY=/app/syncmaster/server/static + server: + static_files: + enabled: True + directory: /app/syncmaster/server/static """ enabled: bool = Field(default=True, description="Set to ``True`` to enable static file serving") @@ -26,7 +29,9 @@ class StaticFilesSettings(BaseModel): ) @field_validator("directory") - def _validate_directory(cls, value: Path) -> Path: + def _validate_directory(cls, value: Path, info: ValidationInfo) -> Path: + if not info.data.get("enabled"): + return value if not value.exists(): raise ValueError(f"Directory '{value}' does not exist") if not value.is_dir(): diff --git a/syncmaster/settings/__init__.py b/syncmaster/settings/__init__.py index 2ccdbf1e..64441769 100644 --- a/syncmaster/settings/__init__.py +++ b/syncmaster/settings/__init__.py @@ -1,13 +1,16 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 +from syncmaster.settings.base import BaseSettings from syncmaster.settings.broker import RabbitMQSettings from syncmaster.settings.credentials import CredentialsEncryptionSettings from syncmaster.settings.database import DatabaseSettings -from syncmaster.settings.log import LoggingSettings +from syncmaster.settings.logging import DEFAULT_LOGGING_SETTINGS, LoggingSettings __all__ = [ - "RabbitMQSettings", + "BaseSettings", "CredentialsEncryptionSettings", "DatabaseSettings", "LoggingSettings", + "DEFAULT_LOGGING_SETTINGS", + "RabbitMQSettings", ] diff --git a/syncmaster/settings/base.py b/syncmaster/settings/base.py new file mode 100644 index 00000000..9eafdf80 --- /dev/null +++ b/syncmaster/settings/base.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +import os + +from pydantic_settings import BaseSettings as PydanticBaseSettings +from pydantic_settings import ( + PydanticBaseSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + + +class BaseSettings(PydanticBaseSettings): + model_config = SettingsConfigDict( + env_prefix="SYNCMASTER__", + env_nested_delimiter="__", + extra="ignore", + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[PydanticBaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + yaml_file_path = os.getenv("SYNCMASTER_CONFIG_FILE", "config.yml") + yaml_settings = YamlConfigSettingsSource(settings_cls, yaml_file=yaml_file_path) + return ( + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + yaml_settings, + ) diff --git a/syncmaster/settings/broker.py b/syncmaster/settings/broker.py index f07a4621..b2ed1a61 100644 --- a/syncmaster/settings/broker.py +++ b/syncmaster/settings/broker.py @@ -11,13 +11,14 @@ class RabbitMQSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - # Set the RabbitMQ connection URL - SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ + broker: + url: amqp://guest:guest@rabbitmq:5672/ - # Pass custom options directly - SYNCMASTER__BROKER__CONNECTION_TIMEOUT=30 + # custom option passed directly to RabbitMQ client + connection_timeout: 30 """ url: str = Field( diff --git a/syncmaster/settings/credentials.py b/syncmaster/settings/credentials.py index 89aba8d5..ed83c477 100644 --- a/syncmaster/settings/credentials.py +++ b/syncmaster/settings/credentials.py @@ -21,10 +21,11 @@ class CredentialsEncryptionSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - # Set the encryption key - SYNCMASTER__ENCRYPTION__SECRET_KEY=secret_key + encryption: + secret_key: UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= """ secret_key: str = Field( diff --git a/syncmaster/settings/database.py b/syncmaster/settings/database.py index 7ccd631c..2c762cc2 100644 --- a/syncmaster/settings/database.py +++ b/syncmaster/settings/database.py @@ -18,11 +18,14 @@ class DatabaseSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster - # custom option passed directly to engine factory - DATABASE_POOL_PRE_PING=True + database: + url: postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster + + # custom option passed directly to SQLAlchemy Engine + pool_pre_ping: True """ url: str = Field( diff --git a/syncmaster/settings/log/__init__.py b/syncmaster/settings/log/__init__.py deleted file mode 100644 index 7b28fedc..00000000 --- a/syncmaster/settings/log/__init__.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 MTS PJSC -# SPDX-License-Identifier: Apache-2.0 - -import logging -import textwrap -from logging.config import dictConfig -from pathlib import Path -from typing import Literal - -import yaml -from pydantic import BaseModel, Field - -LOG_PATH = Path(__file__).parent.resolve() -logger = logging.getLogger(__name__) - - -class LoggingSetupError(Exception): - pass - - -def setup_logging(config_path: Path) -> None: - """Parse file with logging configuration, and setup logging accordingly""" - if not config_path.exists(): - raise OSError(f"Logging configuration file '{config_path}' does not exist") - - try: - config = yaml.safe_load(config_path.read_text()) - dictConfig(config) - except Exception as e: - raise LoggingSetupError(f"Error reading logging configuration '{config_path}'") from e - - -class LoggingSettings(BaseModel): - """Logging Settings. - - Examples - -------- - - Using ``json`` preset: - - .. code-block:: bash - - SYNCMASTER__LOGGING__SETUP=True - SYNCMASTER__LOGGING__PRESET=json - - Passing custom logging config file: - - .. code-block:: bash - - SYNCMASTER__LOGGING__SETUP=True - SYNCMASTER__LOGGING__CUSTOM__CONFIG_PATH=/some/logging.yml - - Setup logging in some other way, e.g. using `uvicorn args `_: - - .. code-block:: bash - - $ export SYNCMASTER__LOGGING__SETUP=False - $ python -m syncmaster.server --log-level debug - """ - - setup: bool = Field( - default=True, - description="If ``True``, setup logging during application start", - ) - preset: Literal["json", "plain", "colored"] = Field( - default="plain", - description=textwrap.dedent( - """ - Name of logging preset to use. - - There are few logging presets bundled to ``syncmaster[server]`` package: - - .. dropdown:: ``plain`` preset - - This preset is recommended to use in environment which do not support colored output, - e.g. CI jobs - - .. literalinclude:: ../../../../syncmaster/settings/log/plain.yml - - .. dropdown:: ``colored`` preset - - This preset is recommended to use in development environment, - as it simplifies debugging. Each log record is output with color specific for a log level - - .. literalinclude:: ../../../../syncmaster/settings/log/colored.yml - - .. dropdown:: ``json`` preset - - This preset is recommended to use in production environment, - as it allows to avoid writing complex log parsing configs. Each log record is output as JSON line - - .. literalinclude:: ../../../../syncmaster/settings/log/json.yml - """, - ), - ) - - custom_config_path: Path | None = Field( - default=None, - description=textwrap.dedent( - """ - Path to custom logging configuration file. If set, overrides :obj:`~preset` value. - - File content should be in YAML format and conform - `logging.dictConfig `_. - """, - ), - ) - - def get_log_config_path(self) -> Path: - return self.custom_config_path or LOG_PATH / f"{self.preset}.yml" diff --git a/syncmaster/settings/log/colored.yml b/syncmaster/settings/log/colored.yml deleted file mode 100644 index 6c0dbeb3..00000000 --- a/syncmaster/settings/log/colored.yml +++ /dev/null @@ -1,57 +0,0 @@ -# development usage only -version: 1 -disable_existing_loggers: false - -filters: - # Add request ID as extra field named `correlation_id` to each log record. - # This is used in combination with settings.server.request_id.enabled=True - # See https://github.com/snok/asgi-correlation-id#configure-logging - correlation_id: - (): asgi_correlation_id.CorrelationIdFilter - uuid_length: 32 - default_value: '-' - -formatters: - colored: - (): coloredlogs.ColoredFormatter - # Add correlation_id to log records - fmt: '%(asctime)s.%(msecs)03d %(processName)s:%(process)d %(name)s:%(lineno)d [%(levelname)s] %(correlation_id)s %(message)s' - datefmt: '%Y-%m-%d %H:%M:%S' - -handlers: - main: - class: logging.StreamHandler - formatter: colored - filters: [correlation_id] - stream: ext://sys.stdout - celery: - class: logging.StreamHandler - formatter: colored - filters: [correlation_id] - stream: ext://sys.stdout - -loggers: - '': - handlers: [main] - level: INFO - propagate: false - uvicorn: - handlers: [main] - level: INFO - propagate: false - celery: - level: INFO - handlers: [celery] - propagate: false - scheduler: - handlers: [main] - level: INFO - propagate: false - py4j: - handlers: [main] - level: WARNING - propagate: false - hdfs.client: - handlers: [main] - level: WARNING - propagate: false diff --git a/syncmaster/settings/log/json.yml b/syncmaster/settings/log/json.yml deleted file mode 100644 index d2916e77..00000000 --- a/syncmaster/settings/log/json.yml +++ /dev/null @@ -1,56 +0,0 @@ -version: 1 -disable_existing_loggers: false - -filters: - # Add request ID as extra field named `correlation_id` to each log record. - # This is used in combination with settings.server.request_id.enabled=True - # See https://github.com/snok/asgi-correlation-id#configure-logging - correlation_id: - (): asgi_correlation_id.CorrelationIdFilter - uuid_length: 32 - default_value: '-' - -formatters: - json: - (): pythonjsonlogger.jsonlogger.JsonFormatter - # Add correlation_id to log records - fmt: '%(processName)s %(process)d %(threadName)s %(thread)d %(name)s %(lineno)d %(levelname)s %(message)s %(correlation_id)s' - timestamp: true - -handlers: - main: - class: logging.StreamHandler - formatter: json - filters: [correlation_id] - stream: ext://sys.stdout - celery: - class: logging.StreamHandler - formatter: json - filters: [correlation_id] - stream: ext://sys.stdout - -loggers: - '': - handlers: [main] - level: INFO - propagate: false - uvicorn: - handlers: [main] - level: INFO - propagate: false - celery: - level: INFO - handlers: [celery] - propagate: false - scheduler: - handlers: [main] - level: INFO - propagate: false - py4j: - handlers: [main] - level: WARNING - propagate: false - hdfs.client: - handlers: [main] - level: WARNING - propagate: false diff --git a/syncmaster/settings/log/plain.yml b/syncmaster/settings/log/plain.yml deleted file mode 100644 index e213a606..00000000 --- a/syncmaster/settings/log/plain.yml +++ /dev/null @@ -1,57 +0,0 @@ -# development usage only -version: 1 -disable_existing_loggers: false - -filters: - # Add request ID as extra field named `correlation_id` to each log record. - # This is used in combination with settings.server.request_id.enabled=True - # See https://github.com/snok/asgi-correlation-id#configure-logging - correlation_id: - (): asgi_correlation_id.CorrelationIdFilter - uuid_length: 32 - default_value: '-' - -formatters: - plain: - (): logging.Formatter - # Add correlation_id to log records - fmt: '%(asctime)s.%(msecs)03d %(processName)s:%(process)d %(name)s:%(lineno)d [%(levelname)s] %(correlation_id)s %(message)s' - datefmt: '%Y-%m-%d %H:%M:%S' - -handlers: - main: - class: logging.StreamHandler - formatter: plain - filters: [correlation_id] - stream: ext://sys.stdout - celery: - class: logging.StreamHandler - formatter: plain - filters: [correlation_id] - stream: ext://sys.stdout - -loggers: - '': - handlers: [main] - level: INFO - propagate: false - uvicorn: - handlers: [main] - level: INFO - propagate: false - celery: - level: INFO - handlers: [celery] - propagate: false - scheduler: - handlers: [main] - level: INFO - propagate: false - py4j: - handlers: [main] - level: WARNING - propagate: false - hdfs.client: - handlers: [main] - level: WARNING - propagate: false diff --git a/syncmaster/settings/logging.py b/syncmaster/settings/logging.py new file mode 100644 index 00000000..bc0ab278 --- /dev/null +++ b/syncmaster/settings/logging.py @@ -0,0 +1,288 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +import logging +import logging.config # noqa: WPS301, WPS458 + +from pydantic import AliasChoices, BaseModel, ConfigDict, Field +from pydantic_settings_logging import ( + FilterConfig, + FormatterConfig, + HandlerConfig, + LoggerConfig, +) +from pydantic_settings_logging import LoggingSettings as BaseLoggingSettings +from pydantic_settings_logging import ( + RootLoggerConfig, + StreamHandlerConfig, +) + + +# https://github.com/vduseev/pydantic-settings-logging/pull/1 +class CallableFactoryConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + callable: str = Field( + description="Custom callable", + validation_alias=AliasChoices("callable", "()"), + serialization_alias="()", + ) + + +class LoggingSettings(BaseLoggingSettings): + """Python logging configuration. + + See `logging.config `_ docs. + + Logging to ``stdout`` with colored text: + + .. code-block:: yaml + :caption: config.yml + + logging: + # development usage only + version: 1 + disable_existing_loggers: false + + filters: + # Add request ID as extra field named `correlation_id` to each log record. + # This is used in combination with settings.server.request_id.enabled=True + # See https://github.com/snok/asgi-correlation-id#configure-logging + correlation_id: + (): asgi_correlation_id.CorrelationIdFilter + uuid_length: 32 + default_value: '-' + + formatters: + colored: + class: coloredlogs.ColoredFormatter + # Add correlation_id to log records + format: '%(asctime)s.%(msecs)03d %(processName)s:%(process)d %(name)s:%(lineno)d [%(levelname)s] %(correlation_id)s %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' + + handlers: + main: + class: logging.StreamHandler + formatter: colored + filters: [correlation_id] + stream: ext://sys.stdout + + root: + handlers: [main] + level: INFO + + loggers: + uvicorn: + handlers: [main] + level: INFO + propagate: false + celery: + handlers: [main] + level: INFO + propagate: false + scheduler: + handlers: [main] + level: INFO + propagate: false + py4j: + handlers: [main] + level: WARNING + propagate: false + hdfs.client: + handlers: [main] + level: WARNING + propagate: false + + + Logging to ``stdout`` without colored text: + + .. code-block:: yaml + :caption: config.yml + + logging: + # development usage only + version: 1 + disable_existing_loggers: false + + filters: + # Add request ID as extra field named `correlation_id` to each log record. + # This is used in combination with settings.server.request_id.enabled=True + # See https://github.com/snok/asgi-correlation-id#configure-logging + correlation_id: + class: asgi_correlation_id.CorrelationIdFilter + uuid_length: 32 + default_value: '-' + + formatters: + plain: + (): logging.Formatter + # Add correlation_id to log records + format: '%(asctime)s.%(msecs)03d %(processName)s:%(process)d %(name)s:%(lineno)d [%(levelname)s] %(correlation_id)s %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' + + handlers: + main: + class: logging.StreamHandler + formatter: plain + filters: [correlation_id] + stream: ext://sys.stdout + + root: + handlers: [main] + level: INFO + + loggers: + uvicorn: + handlers: [main] + level: INFO + propagate: false + celery: + handlers: [main] + level: INFO + propagate: false + scheduler: + handlers: [main] + level: INFO + propagate: false + py4j: + handlers: [main] + level: WARNING + propagate: false + hdfs.client: + handlers: [main] + level: WARNING + propagate: false + + + Logging to ``stdout`` in JSON format: + + .. code-block:: yaml + :caption: config.yml + + logging: + version: 1 + disable_existing_loggers: false + + filters: + # Add request ID as extra field named `correlation_id` to each log record. + # This is used in combination with settings.server.request_id.enabled=True + # See https://github.com/snok/asgi-correlation-id#configure-logging + correlation_id: + (): asgi_correlation_id.CorrelationIdFilter + uuid_length: 32 + default_value: '-' + + formatters: + json: + (): pythonjsonlogger.jsonlogger.JsonFormatter + # Add correlation_id to log records + format: '%(processName)s %(process)d %(threadName)s %(thread)d %(name)s %(lineno)d %(levelname)s %(message)s %(correlation_id)s' + timestamp: true + + handlers: + main: + class: logging.StreamHandler + formatter: json + filters: [correlation_id] + stream: ext://sys.stdout + + root: + handlers: [main] + level: INFO + + loggers: + uvicorn: + handlers: [main] + level: INFO + propagate: false + celery: + handlers: [main] + level: INFO + propagate: false + scheduler: + handlers: [main] + level: INFO + propagate: false + py4j: + handlers: [main] + level: WARNING + propagate: false + hdfs.client: + handlers: [main] + level: WARNING + propagate: false + + """ # noqa: E501 + + filters: dict[str, FilterConfig | CallableFactoryConfig] = Field( + default_factory=dict, + description="Logging filters", + ) + formatters: dict[str, FormatterConfig | CallableFactoryConfig] = Field( + default_factory=dict, + description="Logging formatters", + ) + handlers: dict[str, HandlerConfig | CallableFactoryConfig] = Field( + default_factory=dict, + description="Logging handlers", + ) + + +DEFAULT_LOGGING_SETTINGS = LoggingSettings( + disable_existing_loggers=False, + filters={ + # Add request ID as extra field named `correlation_id` to each log record. + # This is used in combination with settings.server.request_id.enabled=True + # See https://github.com/snok/asgi-correlation-id#configure-logging + "correlation_id": CallableFactoryConfig( + callable="asgi_correlation_id.CorrelationIdFilter", + uuid_length=32, + default_value="-", + ), + }, + formatters={ + "colored": FormatterConfig( + class_="coloredlogs.ColoredFormatter", + format=( + "%(asctime)s.%(msecs)03d %(processName)s:%(process)d %(name)s:%(lineno)d [%(levelname)s] %(correlation_id)s %(message)s" + ), + datefmt="%Y-%m-%d %H:%M:%S", + ), + }, + handlers={ + "main": StreamHandlerConfig( + formatter="colored", + filters=["correlation_id"], + stream="ext://sys.stdout", + ), + }, + root=RootLoggerConfig( + handlers=["main"], + level="INFO", + ), + loggers={ + "uvicorn": LoggerConfig( + handlers=["main"], + level="INFO", + propagate=False, + ), + "celery": LoggerConfig( + handlers=["main"], + level="INFO", + propagate=False, + ), + "py4j": LoggerConfig( + handlers=["main"], + level="WARNING", + propagate=False, + ), + "hdfs.client": LoggerConfig( + handlers=["main"], + level="WARNING", + propagate=False, + ), + }, +) + + +def setup_logging(settings: LoggingSettings): + logging.config.dictConfig(settings.model_dump()) diff --git a/syncmaster/worker/settings/__init__.py b/syncmaster/worker/settings/__init__.py index 63945792..ccd6bb87 100644 --- a/syncmaster/worker/settings/__init__.py +++ b/syncmaster/worker/settings/__init__.py @@ -1,10 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 -from pydantic import Field +from pydantic import BaseModel, Field from pydantic.types import ImportString -from pydantic_settings import BaseSettings, SettingsConfigDict from syncmaster.settings import ( + DEFAULT_LOGGING_SETTINGS, + BaseSettings, CredentialsEncryptionSettings, DatabaseSettings, LoggingSettings, @@ -13,16 +14,18 @@ from syncmaster.worker.settings.hwm_store import HWMStoreSettings -class WorkerSettings(BaseSettings): +class WorkerSettings(BaseModel): """Celery worker settings. Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=custom_syncmaster.spark.get_worker_spark_session - SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }} + worker: + log_url_template: https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }} + create_spark_session_function: custom_syncmaster.spark.get_worker_spark_session """ CREATE_SPARK_SESSION_FUNCTION: ImportString = Field( @@ -40,13 +43,11 @@ class WorkerAppSettings(BaseSettings): Worker application settings. This class is used to configure various settings for the worker application. - The settings can be defined in two ways: + The settings can be passed in several ways: - 1. By explicitly passing a settings object as an argument. - 2. By setting environment variables matching specific keys. - - All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. - Nested items are delimited with ``__``. + 1. By storing settings in a configuration file ``config.yml`` (preferred). + 2. By setting environment variables matching specific keys (``SYNCMASTER__DATABASE__URL`` == ``database.url``). + 3. By explicitly passing a settings object as an argument of application factory function. More details can be found in `Pydantic documentation `_. @@ -54,26 +55,27 @@ class WorkerAppSettings(BaseSettings): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - # Example of setting a database URL via environment variable - SYNCMASTER__DATABASE__URL=postgresql+asyncpg://user:password@localhost:5432/dbname + database: + url: postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster - # Example of setting a broker URL via environment variable - SYNCMASTER__BROKER__URL=amqp://user:password@localhost:5672/ + broker: + url: amqp://user:password@localhost:5672/ - Refer to `Pydantic documentation `_ - for more details on configuration options and environment variable usage. + logging: {} + encryption: {} + worker: {} + hwm_store: {} """ - database: DatabaseSettings = Field(description="Database settings") - broker: RabbitMQSettings = Field(description="Broker settings") - logging: LoggingSettings = Field(default_factory=LoggingSettings, description="Logging settings") - worker: WorkerSettings = Field(default_factory=WorkerSettings, description="Worker-specific settings") + database: DatabaseSettings = Field(default_factory=DatabaseSettings, description="Database settings") + broker: RabbitMQSettings = Field(default_factory=RabbitMQSettings, description="Broker settings") + logging: LoggingSettings = Field(default=DEFAULT_LOGGING_SETTINGS, description="Logging settings") + worker: WorkerSettings = Field(description="Worker-specific settings") encryption: CredentialsEncryptionSettings = Field( default_factory=CredentialsEncryptionSettings, description="Settings for encrypting credential data", ) hwm_store: HWMStoreSettings = Field(default_factory=HWMStoreSettings, description="HWM Store settings") - - model_config = SettingsConfigDict(env_prefix="SYNCMASTER__", env_nested_delimiter="__") diff --git a/syncmaster/worker/settings/hwm_store.py b/syncmaster/worker/settings/hwm_store.py index bd344b9b..79030371 100644 --- a/syncmaster/worker/settings/hwm_store.py +++ b/syncmaster/worker/settings/hwm_store.py @@ -17,15 +17,17 @@ class HWMStoreSettings(BaseModel): Examples -------- - .. code-block:: bash + .. code-block:: yaml + :caption: config.yml - # Set the HWM Store connection URL - SYNCMASTER__HWM_STORE__ENABLED=True - SYNCMASTER__HWM_STORE__TYPE=horizon - SYNCMASTER__HWM_STORE__URL=http://horizon:8000 - SYNCMASTER__HWM_STORE__USER=some_user - SYNCMASTER__HWM_STORE__PASSWORD=changeme - SYNCMASTER__HWM_STORE__NAMESPACE=syncmaster_internal + hwm_store: + # Set the HWM Store connection URL + enabled: true + type: horizon + url: http://horizon:8000 + user: some_user + password: changeme + namespace: syncmaster_internal """ enabled: bool = Field( diff --git a/syncmaster/worker/transfer.py b/syncmaster/worker/transfer.py index 37e16b04..489cdd6b 100644 --- a/syncmaster/worker/transfer.py +++ b/syncmaster/worker/transfer.py @@ -12,7 +12,7 @@ from syncmaster.db.models import AuthData, Run, Status, Transfer from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings.log import setup_logging +from syncmaster.settings.logging import setup_logging from syncmaster.worker.base import WorkerTask from syncmaster.worker.celery import app as celery from syncmaster.worker.controller import TransferController @@ -103,4 +103,4 @@ def run_transfer(run_id: int, engine: Engine, settings: WorkerAppSettings): @after_setup_task_logger.connect def setup_loggers(*args, **kwargs): - setup_logging(WorkerAppSettings().logging.get_log_config_path()) + setup_logging(WorkerAppSettings().logging) diff --git a/tests/test_unit/test_auth/auth_fixtures/__init__.py b/tests/test_unit/test_auth/auth_fixtures/__init__.py index 383638bb..e5d3d908 100644 --- a/tests/test_unit/test_auth/auth_fixtures/__init__.py +++ b/tests/test_unit/test_auth/auth_fixtures/__init__.py @@ -12,6 +12,7 @@ __all__ = [ "create_session_cookie", "mock_keycloak_api", + "mock_keycloak_introspect_token", "mock_keycloak_logout", "mock_keycloak_realm", "mock_keycloak_token_refresh", diff --git a/tests/test_unit/test_auth/auth_fixtures/keycloak_fixture.py b/tests/test_unit/test_auth/auth_fixtures/keycloak_fixture.py index 30d4c993..dfc6699e 100644 --- a/tests/test_unit/test_auth/auth_fixtures/keycloak_fixture.py +++ b/tests/test_unit/test_auth/auth_fixtures/keycloak_fixture.py @@ -86,6 +86,7 @@ def _create_session_cookie(user, expire_in_msec=60000) -> str: @pytest_asyncio.fixture async def mock_keycloak_api(settings): # noqa: F811 + print(settings) keycloak_settings = settings.auth.model_dump()["keycloak"] server_url = keycloak_settings["server_url"] diff --git a/tests/test_unit/test_auth/test_auth_keycloak.py b/tests/test_unit/test_auth/test_keycloak.py similarity index 77% rename from tests/test_unit/test_auth/test_auth_keycloak.py rename to tests/test_unit/test_auth/test_keycloak.py index ed7d89c3..d48c0ffa 100644 --- a/tests/test_unit/test_auth/test_auth_keycloak.py +++ b/tests/test_unit/test_auth/test_keycloak.py @@ -7,21 +7,25 @@ from syncmaster.server.settings import ServerAppSettings as Settings from tests.mocks import MockUser -KEYCLOAK_PROVIDER = "syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider" pytestmark = [pytest.mark.asyncio, pytest.mark.server] - -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": KEYCLOAK_PROVIDER, - }, +KEYCLOAK_AUTH_SETTINGS = { + "auth": { + "provider": "syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider", + "keycloak": { + "server_url": "http://localhost:8080", + "realm_name": "manually_created", + "client_id": "manually_created", + "client_secret": "generated_by_keycloak", + "redirect_uri": "http://localhost:3000/auth/callback", + "scope": "email", + "verify_ssl": False, }, - ], - indirect=True, -) + }, +} + + +@pytest.mark.parametrize("settings", [KEYCLOAK_AUTH_SETTINGS], indirect=True) async def test_keycloak_get_user_unauthorized( client: AsyncClient, simple_user: MockUser, @@ -43,17 +47,7 @@ async def test_keycloak_get_user_unauthorized( @pytest.mark.flaky -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": KEYCLOAK_PROVIDER, - }, - }, - ], - indirect=True, -) +@pytest.mark.parametrize("settings", [KEYCLOAK_AUTH_SETTINGS], indirect=True) async def test_keycloak_get_user_authorized( client: AsyncClient, simple_user: MockUser, @@ -78,17 +72,7 @@ async def test_keycloak_get_user_authorized( } -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": KEYCLOAK_PROVIDER, - }, - }, - ], - indirect=True, -) +@pytest.mark.parametrize("settings", [KEYCLOAK_AUTH_SETTINGS], indirect=True) async def test_keycloak_get_user_expired_access_token( caplog, client: AsyncClient, @@ -116,17 +100,7 @@ async def test_keycloak_get_user_expired_access_token( } -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": KEYCLOAK_PROVIDER, - }, - }, - ], - indirect=True, -) +@pytest.mark.parametrize("settings", [KEYCLOAK_AUTH_SETTINGS], indirect=True) async def test_keycloak_get_user_inactive( client: AsyncClient, simple_user: MockUser, @@ -148,17 +122,7 @@ async def test_keycloak_get_user_inactive( } -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": KEYCLOAK_PROVIDER, - }, - }, - ], - indirect=True, -) +@pytest.mark.parametrize("settings", [KEYCLOAK_AUTH_SETTINGS], indirect=True) async def test_keycloak_auth_callback( client: AsyncClient, settings: Settings, @@ -178,17 +142,7 @@ async def test_keycloak_auth_callback( assert response.status_code == 204, response.text -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": KEYCLOAK_PROVIDER, - }, - }, - ], - indirect=True, -) +@pytest.mark.parametrize("settings", [KEYCLOAK_AUTH_SETTINGS], indirect=True) async def test_keycloak_auth_logout( simple_user: MockUser, client: AsyncClient, diff --git a/tests/test_unit/test_auth/test_oauth2_gateway.py b/tests/test_unit/test_auth/test_oauth2_gateway.py index 248d902a..019d7512 100644 --- a/tests/test_unit/test_auth/test_oauth2_gateway.py +++ b/tests/test_unit/test_auth/test_oauth2_gateway.py @@ -4,21 +4,25 @@ from syncmaster.server.settings import ServerAppSettings as Settings from tests.mocks import MockUser -OAuth2GatewayProvider = "syncmaster.server.providers.auth.oauth2_gateway_provider.OAuth2GatewayProvider" pytestmark = [pytest.mark.asyncio, pytest.mark.server] - -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": OAuth2GatewayProvider, - }, +OAUTH2GATEWAY_AUTH_SETTINGS = { + "auth": { + "provider": "syncmaster.server.providers.auth.oauth2_gateway_provider.OAuth2GatewayProvider", + "keycloak": { + "server_url": "http://localhost:8080", + "realm_name": "manually_created", + "client_id": "manually_created", + "client_secret": "generated_by_keycloak", + "redirect_uri": "http://localhost:3000/auth/callback", + "scope": "email", + "verify_ssl": False, }, - ], - indirect=True, -) + }, +} + + +@pytest.mark.parametrize("settings", [OAUTH2GATEWAY_AUTH_SETTINGS], indirect=True) async def test_get_keycloak_token_active( client: AsyncClient, simple_user: MockUser, @@ -44,17 +48,7 @@ async def test_get_keycloak_token_active( } -@pytest.mark.parametrize( - "settings", - [ - { - "auth": { - "provider": OAuth2GatewayProvider, - }, - }, - ], - indirect=True, -) +@pytest.mark.parametrize("settings", [OAUTH2GATEWAY_AUTH_SETTINGS], indirect=True) async def test_get_keycloak_token_inactive( client: AsyncClient, simple_user: MockUser,