diff --git a/.env.docker.test b/.env.docker.test index 4f5ff47b..a52acbd1 100644 --- a/.env.docker.test +++ b/.env.docker.test @@ -54,8 +54,6 @@ TEST_HIVE_PASSWORD=123UsedForTestOnly@! TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST=http://test-iceberg-rest:8181 TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER=http://test-iceberg-rest:8181 -TEST_ICEBERG_REST_CATALOG_USERNAME=syncmaster -TEST_ICEBERG_REST_CATALOG_PASSWORD=123UsedForTestOnly@! TEST_ICEBERG_S3_WAREHOUSE_PATH=/data TEST_ICEBERG_S3_REGION=us-east-1 TEST_ICEBERG_S3_BUCKET_STYLE=path diff --git a/.env.local.test b/.env.local.test index 444e676b..f114ab07 100644 --- a/.env.local.test +++ b/.env.local.test @@ -54,8 +54,6 @@ export TEST_HIVE_PASSWORD=123UsedForTestOnly@! export TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST=http://localhost:8181 export TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER=http://test-iceberg-rest:8181 -export TEST_ICEBERG_REST_CATALOG_USERNAME=syncmaster -export TEST_ICEBERG_REST_CATALOG_PASSWORD=123UsedForTestOnly@! export TEST_ICEBERG_S3_WAREHOUSE_PATH=/data export TEST_ICEBERG_S3_REGION=us-east-1 export TEST_ICEBERG_S3_BUCKET_STYLE=path diff --git a/syncmaster/dto/connections.py b/syncmaster/dto/connections.py index 14d346c6..3c145c63 100644 --- a/syncmaster/dto/connections.py +++ b/syncmaster/dto/connections.py @@ -96,10 +96,9 @@ class IcebergRESTCatalogS3DirectConnectionBaseDTO(IcebergConnectionBaseDTO): @dataclass(kw_only=True) -class IcebergRESTCatalogBasicAuthS3BasicDTO(IcebergRESTCatalogS3DirectConnectionBaseDTO): - rest_catalog_username: str - rest_catalog_password: str - rest_catalog_auth_type: Literal["basic"] = "basic" +class IcebergRESTCatalogBearerAuthS3BasicDTO(IcebergRESTCatalogS3DirectConnectionBaseDTO): + rest_catalog_token: str + rest_catalog_auth_type: Literal["bearer"] = "bearer" @dataclass(kw_only=True) @@ -121,10 +120,9 @@ class IcebergRESTCatalogS3DelegatedConnectionBaseDTO(IcebergConnectionBaseDTO): @dataclass(kw_only=True) -class IcebergRESTCatalogBasicAuthS3DelegatedDTO(IcebergRESTCatalogS3DelegatedConnectionBaseDTO): - rest_catalog_username: str - rest_catalog_password: str - rest_catalog_auth_type: Literal["basic"] = "basic" +class IcebergRESTCatalogBearerAuthS3DelegatedDTO(IcebergRESTCatalogS3DelegatedConnectionBaseDTO): + rest_catalog_token: str + rest_catalog_auth_type: Literal["bearer"] = "bearer" @dataclass(kw_only=True) @@ -141,18 +139,18 @@ class IcebergRESTCatalogOAuth2ClientCredentialsS3DelegatedDTO(IcebergRESTCatalog # TODO: should be refactored def get_iceberg_rest_catalog_s3_direct_connection_dto( **data, -) -> IcebergRESTCatalogBasicAuthS3BasicDTO | IcebergRESTCatalogOAuth2ClientCredentialsS3BasicDTO: +) -> IcebergRESTCatalogBearerAuthS3BasicDTO | IcebergRESTCatalogOAuth2ClientCredentialsS3BasicDTO: if "rest_catalog_oauth2_client_id" in data: return IcebergRESTCatalogOAuth2ClientCredentialsS3BasicDTO(**data) - return IcebergRESTCatalogBasicAuthS3BasicDTO(**data) + return IcebergRESTCatalogBearerAuthS3BasicDTO(**data) def get_iceberg_rest_catalog_s3_delegated_connection_dto( **data, -) -> IcebergRESTCatalogBasicAuthS3DelegatedDTO | IcebergRESTCatalogOAuth2ClientCredentialsS3DelegatedDTO: +) -> IcebergRESTCatalogBearerAuthS3DelegatedDTO | IcebergRESTCatalogOAuth2ClientCredentialsS3DelegatedDTO: if "rest_catalog_oauth2_client_id" in data: return IcebergRESTCatalogOAuth2ClientCredentialsS3DelegatedDTO(**data) - return IcebergRESTCatalogBasicAuthS3DelegatedDTO(**data) + return IcebergRESTCatalogBearerAuthS3DelegatedDTO(**data) def get_iceberg_connection_dto(**data) -> IcebergConnectionBaseDTO: diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py index afdad3fe..30a408c0 100644 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/__init__.py @@ -4,10 +4,10 @@ from pydantic import Field -from syncmaster.schemas.v1.auth.iceberg_rest_s3_delegated.basic import ( - CreateIcebergRESTCatalogBasicAuthSchema, - ReadIcebergRESTCatalogBasicAuthSchema, - UpdateIcebergRESTCatalogBasicAuthSchema, +from syncmaster.schemas.v1.auth.iceberg_rest_s3_delegated.bearer import ( + CreateIcebergRESTCatalogBearerAuthSchema, + ReadIcebergRESTCatalogBearerAuthSchema, + UpdateIcebergRESTCatalogBearerAuthSchema, ) from syncmaster.schemas.v1.auth.iceberg_rest_s3_delegated.oauth2_client_credentials import ( CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, @@ -16,16 +16,16 @@ ) CreateIcebergRESTCatalogS3DelegatedConnectionAuthDataSchema = Annotated[ - CreateIcebergRESTCatalogBasicAuthSchema | CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, + CreateIcebergRESTCatalogBearerAuthSchema | CreateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, Field(discriminator="type"), ] ReadIcebergRESTCatalogS3DelegatedConnectionAuthDataSchema = Annotated[ - ReadIcebergRESTCatalogBasicAuthSchema | ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, + ReadIcebergRESTCatalogBearerAuthSchema | ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, Field(discriminator="type"), ] UpdateIcebergRESTCatalogS3DelegatedConnectionAuthDataSchema = Annotated[ - UpdateIcebergRESTCatalogBasicAuthSchema | UpdateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, + UpdateIcebergRESTCatalogBearerAuthSchema | UpdateIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, Field(discriminator="type"), ] diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/basic.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/basic.py deleted file mode 100644 index 3f597156..00000000 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/basic.py +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 MTS PJSC -# SPDX-License-Identifier: Apache-2.0 -from typing import Literal - -from pydantic import BaseModel, Field, SecretStr - - -class ReadIcebergRESTCatalogBasicAuthSchema(BaseModel): - type: Literal["iceberg_rest_basic"] = Field(description="Auth type") - rest_catalog_username: str - - -class CreateIcebergRESTCatalogBasicAuthSchema(ReadIcebergRESTCatalogBasicAuthSchema): - rest_catalog_password: SecretStr - - -class UpdateIcebergRESTCatalogBasicAuthSchema(ReadIcebergRESTCatalogBasicAuthSchema): - rest_catalog_password: SecretStr | None = None - - def get_secret_fields(self) -> tuple[str, ...]: - return ("rest_catalog_password",) diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py new file mode 100644 index 00000000..9b03bd6d --- /dev/null +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from typing import Literal + +from pydantic import BaseModel, Field, SecretStr + + +class ReadIcebergRESTCatalogBearerAuthSchema(BaseModel): + type: Literal["iceberg_rest_bearer"] = Field(description="Auth type") + + +class CreateIcebergRESTCatalogBearerAuthSchema(ReadIcebergRESTCatalogBearerAuthSchema): + rest_catalog_token: SecretStr + + +class UpdateIcebergRESTCatalogBearerAuthSchema(ReadIcebergRESTCatalogBearerAuthSchema): + rest_catalog_token: SecretStr | None = None + + def get_secret_fields(self) -> tuple[str, ...]: + return ("rest_catalog_token",) diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py index c3c0115c..9dd1cbd4 100644 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/__init__.py @@ -4,10 +4,10 @@ from pydantic import Field -from syncmaster.schemas.v1.auth.iceberg_rest_s3_direct.basic import ( - CreateIcebergRESTCatalogBasicS3BasicAuthSchema, - ReadIcebergRESTCatalogBasicS3BasicAuthSchema, - UpdateIcebergRESTCatalogBasicS3BasicAuthSchema, +from syncmaster.schemas.v1.auth.iceberg_rest_s3_direct.bearer import ( + CreateIcebergRESTCatalogBearerS3BasicAuthSchema, + ReadIcebergRESTCatalogBearerS3BasicAuthSchema, + UpdateIcebergRESTCatalogBearerS3BasicAuthSchema, ) from syncmaster.schemas.v1.auth.iceberg_rest_s3_direct.oauth2_client_credentials import ( CreateIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, @@ -16,16 +16,16 @@ ) CreateIcebergRESTCatalogS3DirectConnectionAuthDataSchema = Annotated[ - CreateIcebergRESTCatalogBasicS3BasicAuthSchema | CreateIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, + CreateIcebergRESTCatalogBearerS3BasicAuthSchema | CreateIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, Field(discriminator="type"), ] ReadIcebergRESTCatalogS3DirectConnectionAuthDataSchema = Annotated[ - ReadIcebergRESTCatalogBasicS3BasicAuthSchema | ReadIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, + ReadIcebergRESTCatalogBearerS3BasicAuthSchema | ReadIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, Field(discriminator="type"), ] UpdateIcebergRESTCatalogS3DirectConnectionAuthDataSchema = Annotated[ - UpdateIcebergRESTCatalogBasicS3BasicAuthSchema | UpdateIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, + UpdateIcebergRESTCatalogBearerS3BasicAuthSchema | UpdateIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, Field(discriminator="type"), ] diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/basic.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/basic.py deleted file mode 100644 index 9e40b068..00000000 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/basic.py +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 MTS PJSC -# SPDX-License-Identifier: Apache-2.0 -from typing import Literal - -from pydantic import BaseModel, Field, SecretStr - - -class ReadIcebergRESTCatalogBasicS3BasicAuthSchema(BaseModel): - type: Literal["iceberg_rest_basic_s3_basic"] = Field(description="Auth type") - rest_catalog_username: str - s3_access_key: str - - -class CreateIcebergRESTCatalogBasicS3BasicAuthSchema(ReadIcebergRESTCatalogBasicS3BasicAuthSchema): - rest_catalog_password: SecretStr - s3_secret_key: SecretStr - - -class UpdateIcebergRESTCatalogBasicS3BasicAuthSchema(ReadIcebergRESTCatalogBasicS3BasicAuthSchema): - rest_catalog_password: SecretStr | None = None - s3_secret_key: SecretStr | None = None - - def get_secret_fields(self) -> tuple[str, ...]: - return ("rest_catalog_password", "s3_secret_key") diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py new file mode 100644 index 00000000..a4281b4c --- /dev/null +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from typing import Literal + +from pydantic import BaseModel, Field, SecretStr + + +class ReadIcebergRESTCatalogBearerS3BasicAuthSchema(BaseModel): + type: Literal["iceberg_rest_bearer_s3_basic"] = Field(description="Auth type") + s3_access_key: str + + +class CreateIcebergRESTCatalogBearerS3BasicAuthSchema(ReadIcebergRESTCatalogBearerS3BasicAuthSchema): + rest_catalog_token: SecretStr + s3_secret_key: SecretStr + + +class UpdateIcebergRESTCatalogBearerS3BasicAuthSchema(ReadIcebergRESTCatalogBearerS3BasicAuthSchema): + rest_catalog_token: SecretStr | None = None + s3_secret_key: SecretStr | None = None + + def get_secret_fields(self) -> tuple[str, ...]: + return ("rest_catalog_token", "s3_secret_key") diff --git a/syncmaster/schemas/v1/connections/connection_base.py b/syncmaster/schemas/v1/connections/connection_base.py index 02cb7c12..0c91744b 100644 --- a/syncmaster/schemas/v1/connections/connection_base.py +++ b/syncmaster/schemas/v1/connections/connection_base.py @@ -8,11 +8,11 @@ ReadSambaAuthSchema, ) from syncmaster.schemas.v1.auth.iceberg_rest_s3_delegated import ( - ReadIcebergRESTCatalogBasicAuthSchema, + ReadIcebergRESTCatalogBearerAuthSchema, ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema, ) from syncmaster.schemas.v1.auth.iceberg_rest_s3_direct import ( - ReadIcebergRESTCatalogBasicS3BasicAuthSchema, + ReadIcebergRESTCatalogBearerS3BasicAuthSchema, ReadIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema, ) from syncmaster.schemas.v1.types import NameConstr @@ -21,9 +21,9 @@ ReadBasicAuthSchema | ReadS3AuthSchema | ReadSambaAuthSchema - | ReadIcebergRESTCatalogBasicS3BasicAuthSchema + | ReadIcebergRESTCatalogBearerS3BasicAuthSchema | ReadIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema - | ReadIcebergRESTCatalogBasicAuthSchema + | ReadIcebergRESTCatalogBearerAuthSchema | ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema ) diff --git a/syncmaster/worker/handlers/db/iceberg.py b/syncmaster/worker/handlers/db/iceberg.py index 8a3eb519..4a8ba72b 100644 --- a/syncmaster/worker/handlers/db/iceberg.py +++ b/syncmaster/worker/handlers/db/iceberg.py @@ -9,8 +9,8 @@ from onetl.hooks import slot, support_hooks from syncmaster.dto.connections import ( - IcebergRESTCatalogBasicAuthS3BasicDTO, - IcebergRESTCatalogBasicAuthS3DelegatedDTO, + IcebergRESTCatalogBearerAuthS3BasicDTO, + IcebergRESTCatalogBearerAuthS3DelegatedDTO, IcebergRESTCatalogOAuth2ClientCredentialsS3BasicDTO, IcebergRESTCatalogOAuth2ClientCredentialsS3DelegatedDTO, IcebergRESTCatalogS3DelegatedConnectionBaseDTO, @@ -126,10 +126,9 @@ def _make_auth(self): ) if isinstance( self.connection_dto, - IcebergRESTCatalogBasicAuthS3DelegatedDTO | IcebergRESTCatalogBasicAuthS3BasicDTO, + IcebergRESTCatalogBearerAuthS3DelegatedDTO | IcebergRESTCatalogBearerAuthS3BasicDTO, ): - return Iceberg.RESTCatalog.BasicAuth( - user=self.connection_dto.rest_catalog_username, - password=self.connection_dto.rest_catalog_password, + return Iceberg.RESTCatalog.BearerAuth( + access_token=self.connection_dto.rest_catalog_token, ) return None diff --git a/tests/settings.py b/tests/settings.py index 0fe522bb..1abec9d8 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -59,8 +59,6 @@ class TestSettings(BaseSettings): TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST: str TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER: str - TEST_ICEBERG_REST_CATALOG_USERNAME: str - TEST_ICEBERG_REST_CATALOG_PASSWORD: str TEST_ICEBERG_S3_WAREHOUSE_PATH: str TEST_ICEBERG_S3_REGION: str TEST_ICEBERG_S3_BUCKET_STYLE: Literal["domain", "path"] = "path" diff --git a/tests/test_integration/test_run_transfer/connection_fixtures/iceberg_fixtures.py b/tests/test_integration/test_run_transfer/connection_fixtures/iceberg_fixtures.py index fa118ee0..6b4ab211 100644 --- a/tests/test_integration/test_run_transfer/connection_fixtures/iceberg_fixtures.py +++ b/tests/test_integration/test_run_transfer/connection_fixtures/iceberg_fixtures.py @@ -9,7 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from syncmaster.db.models import Group -from syncmaster.dto.connections import IcebergRESTCatalogBasicAuthS3BasicDTO +from syncmaster.dto.connections import IcebergRESTCatalogBearerAuthS3BasicDTO from syncmaster.server.settings import ServerAppSettings as Settings from tests.settings import TestSettings from tests.test_unit.utils import create_connection, create_credentials @@ -21,8 +21,8 @@ scope="session", params=[pytest.param("iceberg", marks=[pytest.mark.iceberg])], ) -def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCatalogBasicAuthS3BasicDTO: - return IcebergRESTCatalogBasicAuthS3BasicDTO( +def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCatalogBearerAuthS3BasicDTO: + return IcebergRESTCatalogBearerAuthS3BasicDTO( rest_catalog_url=test_settings.TEST_ICEBERG_REST_CATALOG_URL_FOR_CONFTEST, s3_warehouse_path=test_settings.TEST_ICEBERG_S3_WAREHOUSE_PATH, s3_host=test_settings.TEST_S3_HOST_FOR_CONFTEST, @@ -34,8 +34,7 @@ def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCata s3_access_key=test_settings.TEST_S3_ACCESS_KEY, s3_secret_key=test_settings.TEST_S3_SECRET_KEY, s3_additional_params=test_settings.TEST_ICEBERG_S3_ADDITIONAL_PARAMS, - rest_catalog_username=test_settings.TEST_ICEBERG_REST_CATALOG_USERNAME, - rest_catalog_password=test_settings.TEST_ICEBERG_REST_CATALOG_PASSWORD, + rest_catalog_token="", ) @@ -43,8 +42,8 @@ def iceberg_rest_s3_for_conftest(test_settings: TestSettings) -> IcebergRESTCata scope="session", params=[pytest.param("iceberg", marks=[pytest.mark.iceberg])], ) -def iceberg_rest_s3_for_worker(test_settings: TestSettings) -> IcebergRESTCatalogBasicAuthS3BasicDTO: - return IcebergRESTCatalogBasicAuthS3BasicDTO( +def iceberg_rest_s3_for_worker(test_settings: TestSettings) -> IcebergRESTCatalogBearerAuthS3BasicDTO: + return IcebergRESTCatalogBearerAuthS3BasicDTO( rest_catalog_url=test_settings.TEST_ICEBERG_REST_CATALOG_URL_FOR_WORKER, s3_warehouse_path=test_settings.TEST_ICEBERG_S3_WAREHOUSE_PATH, s3_host=test_settings.TEST_S3_HOST_FOR_WORKER, @@ -56,15 +55,14 @@ def iceberg_rest_s3_for_worker(test_settings: TestSettings) -> IcebergRESTCatalo s3_access_key=test_settings.TEST_S3_ACCESS_KEY, s3_secret_key=test_settings.TEST_S3_SECRET_KEY, s3_additional_params=test_settings.TEST_ICEBERG_S3_ADDITIONAL_PARAMS, - rest_catalog_username=test_settings.TEST_ICEBERG_REST_CATALOG_USERNAME, - rest_catalog_password=test_settings.TEST_ICEBERG_REST_CATALOG_PASSWORD, + rest_catalog_token="", ) @pytest.fixture def prepare_iceberg_rest_s3( spark: SparkSession, - iceberg_rest_s3_for_conftest: IcebergRESTCatalogBasicAuthS3BasicDTO, + iceberg_rest_s3_for_conftest: IcebergRESTCatalogBearerAuthS3BasicDTO, s3_file_connection: S3, ): iceberg = iceberg_rest_s3_for_conftest @@ -76,13 +74,7 @@ def prepare_iceberg_rest_s3( connection = Iceberg( spark=spark, catalog_name=catalog_name, - catalog=Iceberg.RESTCatalog( - url=iceberg.rest_catalog_url, - auth=Iceberg.RESTCatalog.BasicAuth( - user=iceberg.rest_catalog_username, - password=iceberg.rest_catalog_password, - ), - ), + catalog=Iceberg.RESTCatalog(url=iceberg.rest_catalog_url), warehouse=Iceberg.S3Warehouse( path=iceberg.s3_warehouse_path, host=iceberg.s3_host, @@ -118,7 +110,7 @@ def fill_with_data(df: DataFrame): @pytest_asyncio.fixture async def iceberg_rest_s3_connection( - iceberg_rest_s3_for_worker: IcebergRESTCatalogBasicAuthS3BasicDTO, + iceberg_rest_s3_for_worker: IcebergRESTCatalogBearerAuthS3BasicDTO, settings: Settings, session: AsyncSession, group: Group, @@ -147,11 +139,10 @@ async def iceberg_rest_s3_connection( settings=settings, connection_id=result.id, auth_data=dict( - type="iceberg_rest_basic_s3_basic", + type="iceberg_rest_bearer_s3_basic", s3_access_key=iceberg.s3_access_key, s3_secret_key=iceberg.s3_secret_key, - rest_catalog_username=iceberg.rest_catalog_username, - rest_catalog_password=iceberg.rest_catalog_password, + rest_catalog_token=iceberg.rest_catalog_token, ), ) diff --git a/tests/test_integration/test_run_transfer/test_iceberg.py b/tests/test_integration/test_run_transfer/test_iceberg.py index 9a52c145..03055756 100644 --- a/tests/test_integration/test_run_transfer/test_iceberg.py +++ b/tests/test_integration/test_run_transfer/test_iceberg.py @@ -115,7 +115,7 @@ async def test_run_transfer_postgres_to_iceberg_rest_s3_with_full_strategy( client, group_owner, postgres_to_iceberg_rest_s3.id, - target_auth="iceberg_rest_basic_s3_basic", + target_auth="iceberg_rest_bearer_s3_basic", ) reader = DBReader( @@ -155,7 +155,7 @@ async def test_run_transfer_postgres_to_iceberg_rest_s3_mixed_naming_with_full_s client, group_owner, postgres_to_iceberg_rest_s3.id, - target_auth="iceberg_rest_basic_s3_basic", + target_auth="iceberg_rest_bearer_s3_basic", ) reader = DBReader( @@ -200,7 +200,7 @@ async def test_run_transfer_postgres_to_iceberg_rest_s3_with_incremental_strateg client, group_owner, postgres_to_iceberg_rest_s3.id, - target_auth="iceberg_rest_basic_s3_basic", + target_auth="iceberg_rest_bearer_s3_basic", ) reader = DBReader( @@ -217,7 +217,7 @@ async def test_run_transfer_postgres_to_iceberg_rest_s3_with_incremental_strateg client, group_owner, postgres_to_iceberg_rest_s3.id, - target_auth="iceberg_rest_basic_s3_basic", + target_auth="iceberg_rest_bearer_s3_basic", ) spark.catalog.refreshTable("iceberg_rest_s3.default.target_table") @@ -264,7 +264,7 @@ async def test_run_transfer_iceberg_rest_s3_to_postgres_with_full_strategy( client, group_owner, iceberg_rest_s3_to_postgres.id, - source_auth="iceberg_rest_basic_s3_basic", + source_auth="iceberg_rest_bearer_s3_basic", ) reader = DBReader( @@ -304,7 +304,7 @@ async def test_run_transfer_iceberg_rest_s3_to_postgres_mixes_naming_with_full_s client, group_owner, iceberg_rest_s3_to_postgres.id, - source_auth="iceberg_rest_basic_s3_basic", + source_auth="iceberg_rest_bearer_s3_basic", ) reader = DBReader( @@ -348,7 +348,7 @@ async def test_run_transfer_iceberg_rest_s3_to_postgres_with_incremental_strateg client, group_owner, iceberg_rest_s3_to_postgres.id, - source_auth="iceberg_rest_basic_s3_basic", + source_auth="iceberg_rest_bearer_s3_basic", ) reader = DBReader( @@ -365,7 +365,7 @@ async def test_run_transfer_iceberg_rest_s3_to_postgres_with_incremental_strateg client, group_owner, iceberg_rest_s3_to_postgres.id, - source_auth="iceberg_rest_basic_s3_basic", + source_auth="iceberg_rest_bearer_s3_basic", ) df_with_increment = reader.run() diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_iceberg_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_iceberg_connection.py index 86797f27..12995a84 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_iceberg_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_iceberg_connection.py @@ -40,9 +40,8 @@ async def test_developer_plus_can_create_iceberg_rest_s3_direct_connection( "s3_bucket_style": "path", }, "auth_data": { - "type": "iceberg_rest_basic_s3_basic", - "rest_catalog_username": "user", - "rest_catalog_password": "secret", + "type": "iceberg_rest_bearer_s3_basic", + "rest_catalog_token": "token", "s3_access_key": "access_key", "s3_secret_key": "secret_key", }, @@ -87,7 +86,6 @@ async def test_developer_plus_can_create_iceberg_rest_s3_direct_connection( }, "auth_data": { "type": decrypted["type"], - "rest_catalog_username": decrypted["rest_catalog_username"], "s3_access_key": decrypted["s3_access_key"], }, } @@ -116,9 +114,8 @@ async def test_developer_plus_can_create_iceberg_rest_s3_delegated_connection( "s3_access_delegation": "vended-credentials", }, "auth_data": { - "type": "iceberg_rest_basic", - "rest_catalog_username": "user", - "rest_catalog_password": "secret", + "type": "iceberg_rest_bearer", + "rest_catalog_token": "token", }, }, ) @@ -155,7 +152,6 @@ async def test_developer_plus_can_create_iceberg_rest_s3_delegated_connection( }, "auth_data": { "type": decrypted["type"], - "rest_catalog_username": decrypted["rest_catalog_username"], }, } @@ -328,9 +324,8 @@ async def test_developer_plus_can_create_iceberg_rest_s3_delegated_connection_wi [ pytest.param( { - "type": "iceberg_rest_basic", - "rest_catalog_username": "user", - "rest_catalog_password": "secret", + "type": "iceberg_rest_bearer", + "rest_catalog_token": "token", }, id="with_basic_auth", ), @@ -403,9 +398,8 @@ async def test_developer_plus_can_create_iceberg_rest_s3_direct_connection_witho [ pytest.param( { - "type": "iceberg_rest_basic_s3_basic", - "rest_catalog_username": "user", - "rest_catalog_password": "secret", + "type": "iceberg_rest_bearer_s3_basic", + "rest_catalog_token": "token", "s3_access_key": "access_key", "s3_secret_key": "secret_key", }, diff --git a/tests/test_unit/test_connections/test_db_connection/test_update_iceberg_connection.py b/tests/test_unit/test_connections/test_db_connection/test_update_iceberg_connection.py index 9d64b5cf..a22beb04 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_update_iceberg_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_update_iceberg_connection.py @@ -24,9 +24,8 @@ "s3_bucket_style": "path", }, { - "type": "iceberg_rest_basic_s3_basic", - "rest_catalog_username": "user", - "rest_catalog_password": "secret", + "type": "iceberg_rest_bearer_s3_basic", + "rest_catalog_token": "token", "s3_access_key": "access_key", "s3_secret_key": "secret_key", }, @@ -59,9 +58,8 @@ async def test_developer_plus_can_update_iceberg_rest_s3_direct_connection( "s3_bucket_style": "domain", }, "auth_data": { - "type": "iceberg_rest_basic_s3_basic", - "rest_catalog_username": "new_user", - "rest_catalog_password": "new_password", + "type": "iceberg_rest_bearer_s3_basic", + "rest_catalog_token": "new_token", "s3_access_key": "new_access_key", }, }, @@ -88,7 +86,6 @@ async def test_developer_plus_can_update_iceberg_rest_s3_direct_connection( }, "auth_data": { "type": group_connection.credentials.value["type"], - "rest_catalog_username": "new_user", "s3_access_key": "new_access_key", }, } @@ -106,9 +103,8 @@ async def test_developer_plus_can_update_iceberg_rest_s3_direct_connection( "s3_access_delegation": "vended-credentials", }, { - "type": "iceberg_rest_basic", - "rest_catalog_username": "user", - "rest_catalog_password": "secret", + "type": "iceberg_rest_bearer", + "rest_catalog_token": "token", }, ), ], @@ -135,9 +131,8 @@ async def test_developer_plus_can_update_iceberg_rest_s3_delegated_connection( "s3_access_delegation": "remote-signing", }, "auth_data": { - "type": "iceberg_rest_basic", - "rest_catalog_username": "new_user", - "rest_catalog_password": "new_password", + "type": "iceberg_rest_bearer", + "rest_catalog_token": "new_token", }, }, ) @@ -156,8 +151,7 @@ async def test_developer_plus_can_update_iceberg_rest_s3_delegated_connection( "s3_access_delegation": "remote-signing", }, "auth_data": { - "type": "iceberg_rest_basic", - "rest_catalog_username": "new_user", + "type": "iceberg_rest_bearer", }, } diff --git a/tests/test_unit/utils.py b/tests/test_unit/utils.py index 041fd30f..3f424d8c 100644 --- a/tests/test_unit/utils.py +++ b/tests/test_unit/utils.py @@ -276,8 +276,8 @@ async def fetch_connection_json(client: AsyncClient, user_token: str, mock_conne "basic": ["password"], "samba": ["password"], "s3": ["secret_key"], - "iceberg_rest_basic": ["rest_catalog_password"], - "iceberg_rest_basic_s3_basic": ["rest_catalog_password", "s3_secret_key"], + "iceberg_rest_bearer": ["rest_catalog_token"], + "iceberg_rest_bearer_s3_basic": ["rest_catalog_token", "s3_secret_key"], "iceberg_rest_oauth2_client_credentials": ["rest_catalog_oauth2_client_secret"], "iceberg_rest_oauth2_client_credentials_s3_basic": [ "rest_catalog_oauth2_client_secret", diff --git a/tests/utils.py b/tests/utils.py index fbdee997..6fca58f5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -140,14 +140,12 @@ def verify_transfer_auth_data(run_data: dict[str, Any], source_auth: str, target if auth_type == "s3": assert auth_data["access_key"] assert "secret_key" not in auth_data - elif auth_type == "iceberg_rest_basic": - assert auth_data["rest_catalog_username"] - assert "rest_catalog_password" not in auth_data - elif auth_type == "iceberg_rest_basic_s3_basic": + elif auth_type == "iceberg_rest_bearer": + assert "rest_catalog_token" not in auth_data + elif auth_type == "iceberg_rest_bearer_s3_basic": assert auth_data["s3_access_key"] - assert auth_data["rest_catalog_username"] assert "s3_secret_key" not in auth_data - assert "rest_catalog_password" not in auth_data + assert "rest_catalog_token" not in auth_data elif auth_type == "iceberg_rest_oauth2_client_credentials": assert auth_data["rest_catalog_oauth2_client_id"] assert "rest_catalog_oauth2_client_secret" not in auth_data