diff --git a/.gitignore b/.gitignore index ffc47b9c..ec56ced9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ runs/ model.safetensors training_args.bin /temp/ +.devcontainer/.env.secrets +.devcontainer/ diff --git a/alembic/versions/2026_02_26_2006-94e2b850fb30_remove_agency_described_not_in_database_.py b/alembic/versions/2026_02_26_2006-94e2b850fb30_remove_agency_described_not_in_database_.py new file mode 100644 index 00000000..03d7e843 --- /dev/null +++ b/alembic/versions/2026_02_26_2006-94e2b850fb30_remove_agency_described_not_in_database_.py @@ -0,0 +1,32 @@ +"""remove agency_described_not_in_database from url optional metadata + +Revision ID: 94e2b850fb30 +Revises: 1fb2286a016c +Create Date: 2026-02-26 20:06:49.423584 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '94e2b850fb30' +down_revision: Union[str, None] = '1fb2286a016c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_column( + 'url_optional_data_source_metadata', + 'agency_described_not_in_database', + ) + + +def downgrade() -> None: + op.add_column( + 'url_optional_data_source_metadata', + sa.Column('agency_described_not_in_database', sa.Text(), nullable=True), + ) diff --git a/src/api/endpoints/data_source/_shared/build.py b/src/api/endpoints/data_source/_shared/build.py index 35b65343..a4383716 100644 --- a/src/api/endpoints/data_source/_shared/build.py +++ b/src/api/endpoints/data_source/_shared/build.py @@ -30,7 +30,6 @@ def build_data_source_get_query() -> Select: URLOptionalDataSourceMetadata.coverage_end, URLOptionalDataSourceMetadata.agency_supplied, URLOptionalDataSourceMetadata.agency_aggregation, - URLOptionalDataSourceMetadata.agency_described_not_in_database, URLOptionalDataSourceMetadata.agency_originated, URLOptionalDataSourceMetadata.update_method, URLOptionalDataSourceMetadata.readme_url, @@ -63,4 +62,4 @@ def build_data_source_get_query() -> Select: .options( selectinload(URL.confirmed_agencies), ) - ) \ No newline at end of file + ) diff --git a/src/api/endpoints/data_source/_shared/process.py b/src/api/endpoints/data_source/_shared/process.py index 252ed7c0..dd896a94 100644 --- a/src/api/endpoints/data_source/_shared/process.py +++ b/src/api/endpoints/data_source/_shared/process.py @@ -32,7 +32,6 @@ def process_data_source_get_mapping( agency_supplied=mapping[URLOptionalDataSourceMetadata.agency_supplied], agency_aggregation=mapping[URLOptionalDataSourceMetadata.agency_aggregation], agency_originated=mapping[URLOptionalDataSourceMetadata.agency_originated], - agency_described_not_in_database=mapping[URLOptionalDataSourceMetadata.agency_described_not_in_database], update_method=mapping[URLOptionalDataSourceMetadata.update_method], readme_url=mapping[URLOptionalDataSourceMetadata.readme_url], originating_entity=mapping[URLOptionalDataSourceMetadata.originating_entity], @@ -41,4 +40,4 @@ def process_data_source_get_mapping( submission_notes=mapping[URLOptionalDataSourceMetadata.submission_notes], access_notes=mapping[URLOptionalDataSourceMetadata.access_notes], access_types=mapping[URLOptionalDataSourceMetadata.access_types] or [] - ) \ No newline at end of file + ) diff --git a/src/api/endpoints/data_source/by_id/put/query.py b/src/api/endpoints/data_source/by_id/put/query.py index 96106395..76f1c138 100644 --- a/src/api/endpoints/data_source/by_id/put/query.py +++ b/src/api/endpoints/data_source/by_id/put/query.py @@ -70,8 +70,6 @@ async def run(self, session: AsyncSession) -> None: value_dict["agency_originated"] = self.request.agency_originated if self.request.agency_aggregation is not None: value_dict["agency_aggregation"] = self.request.agency_aggregation - if self.request.agency_described_not_in_database is not None: - value_dict["agency_described_not_in_database"] = self.request.agency_described_not_in_database if self.request.update_method is not None: value_dict["update_method"] = self.request.update_method if self.request.readme_url is not None: @@ -120,4 +118,3 @@ async def run(self, session: AsyncSession) -> None: await session.execute(statement) - diff --git a/src/api/endpoints/data_source/by_id/put/request.py b/src/api/endpoints/data_source/by_id/put/request.py index 28549c28..d1ded786 100644 --- a/src/api/endpoints/data_source/by_id/put/request.py +++ b/src/api/endpoints/data_source/by_id/put/request.py @@ -27,7 +27,6 @@ class DataSourcePutRequest(BaseModel): agency_supplied: bool | None = None agency_originated: bool | None = None agency_aggregation: AgencyAggregationEnum | None = None - agency_described_not_in_database: str | None = None update_method: UpdateMethodEnum | None = None readme_url: str | None = None originating_entity: str | None = None @@ -47,7 +46,6 @@ def optional_data_source_metadata_not_none(self) -> bool: self.agency_supplied is not None or self.agency_originated is not None or self.agency_aggregation is not None or - self.agency_described_not_in_database is not None or self.update_method is not None or self.readme_url is not None or self.originating_entity is not None or @@ -56,4 +54,4 @@ def optional_data_source_metadata_not_none(self) -> bool: self.submission_notes is not None or self.access_notes is not None or self.access_types is not None - ) \ No newline at end of file + ) diff --git a/src/api/endpoints/data_source/get/response.py b/src/api/endpoints/data_source/get/response.py index b80ee9e1..6356e21d 100644 --- a/src/api/endpoints/data_source/get/response.py +++ b/src/api/endpoints/data_source/get/response.py @@ -29,7 +29,6 @@ class DataSourceGetResponse(BaseModel): agency_supplied: bool | None = None agency_originated: bool | None = None agency_aggregation: AgencyAggregationEnum | None = None - agency_described_not_in_database: str | None = None update_method: UpdateMethodEnum | None = None readme_url: str | None = None originating_entity: str | None = None diff --git a/src/api/endpoints/submit/data_source/queries/core.py b/src/api/endpoints/submit/data_source/queries/core.py index aec2e821..eed12aca 100644 --- a/src/api/endpoints/submit/data_source/queries/core.py +++ b/src/api/endpoints/submit/data_source/queries/core.py @@ -129,7 +129,6 @@ async def run( agency_supplied=self.request.agency_supplied, agency_originated=self.request.agency_originated, agency_aggregation=self.request.agency_aggregation, - agency_described_not_in_database=self.request.agency_described_not_in_database, data_portal_type=self.request.data_portal_type, update_method=self.request.update_method, readme_url=self.request.readme_url, @@ -146,4 +145,4 @@ async def run( return SubmitDataSourceURLProposalResponse( url_id=url_id, - ) \ No newline at end of file + ) diff --git a/src/api/endpoints/submit/data_source/request.py b/src/api/endpoints/submit/data_source/request.py index fe541923..936d34b6 100644 --- a/src/api/endpoints/submit/data_source/request.py +++ b/src/api/endpoints/submit/data_source/request.py @@ -20,7 +20,6 @@ class DataSourceSubmissionRequest(RequestBase): agency_supplied: bool | None = None agency_originated: bool | None = None agency_aggregation: AgencyAggregationEnum | None = None - agency_described_not_in_database: str | None = None data_portal_type: str | None = None update_method: UpdateMethodEnum | None = None readme_url: str | None = None diff --git a/src/db/models/impl/url/optional_ds_metadata/sqlalchemy.py b/src/db/models/impl/url/optional_ds_metadata/sqlalchemy.py index 04541ad6..bee333cd 100644 --- a/src/db/models/impl/url/optional_ds_metadata/sqlalchemy.py +++ b/src/db/models/impl/url/optional_ds_metadata/sqlalchemy.py @@ -26,7 +26,6 @@ class URLOptionalDataSourceMetadata( agency_supplied = Column(Boolean, nullable=True) agency_originated = Column(Boolean, nullable=True) agency_aggregation: Mapped[AgencyAggregationEnum] = enum_column(AgencyAggregationEnum, name="agency_aggregation_enum") - agency_described_not_in_database = Column(String, nullable=True) update_method: Mapped[UpdateMethodEnum] = enum_column(UpdateMethodEnum, name="update_method_enum") readme_url = Column(String, nullable=True) originating_entity = Column(String, nullable=True) diff --git a/src/external/pdap/impl/sync/data_sources/_shared/content.py b/src/external/pdap/impl/sync/data_sources/_shared/content.py index 59d0bcc6..5be3529b 100644 --- a/src/external/pdap/impl/sync/data_sources/_shared/content.py +++ b/src/external/pdap/impl/sync/data_sources/_shared/content.py @@ -28,7 +28,6 @@ class DataSourceSyncContentModel(BaseModel): agency_supplied: bool | None = None agency_originated: bool | None = None agency_aggregation: AgencyAggregationEnum | None = None - agency_described_not_in_database: str | None = None update_method: UpdateMethodEnum | None = None readme_url: str | None = None originating_entity: str | None = None diff --git a/tests/automated/integration/api/data_sources/test_put.py b/tests/automated/integration/api/data_sources/test_put.py index c954b59c..aece2075 100644 --- a/tests/automated/integration/api/data_sources/test_put.py +++ b/tests/automated/integration/api/data_sources/test_put.py @@ -39,7 +39,6 @@ async def test_put( agency_supplied=False, agency_originated=True, agency_aggregation=AgencyAggregationEnum.LOCALITY, - agency_described_not_in_database="Modified Agency Not In DB", update_method=UpdateMethodEnum.OVERWRITE, readme_url="https://modified-readme.com", originating_entity="Modified Originating Entity", @@ -78,7 +77,6 @@ async def test_put( assert optional_metadata.agency_supplied == False assert optional_metadata.agency_originated == True assert optional_metadata.agency_aggregation == AgencyAggregationEnum.LOCALITY - assert optional_metadata.agency_described_not_in_database == "Modified Agency Not In DB" assert optional_metadata.update_method == UpdateMethodEnum.OVERWRITE assert optional_metadata.readme_url == "https://modified-readme.com" assert optional_metadata.originating_entity == "Modified Originating Entity" diff --git a/tests/automated/integration/api/submit/data_source/test_core.py b/tests/automated/integration/api/submit/data_source/test_core.py index bf339bfd..f616ac7b 100644 --- a/tests/automated/integration/api/submit/data_source/test_core.py +++ b/tests/automated/integration/api/submit/data_source/test_core.py @@ -43,7 +43,6 @@ async def test_submit_data_source( agency_supplied=True, agency_originated=False, agency_aggregation=AgencyAggregationEnum.STATE, - agency_described_not_in_database="Test agency described not in database", update_method=UpdateMethodEnum.NO_UPDATES, readme_url="https://example.com/readme", originating_entity="Test Originating Entity", @@ -131,7 +130,6 @@ async def test_submit_data_source( assert optional_ds.agency_supplied assert not optional_ds.agency_originated assert optional_ds.agency_aggregation == AgencyAggregationEnum.STATE - assert optional_ds.agency_described_not_in_database == "Test agency described not in database" assert optional_ds.data_portal_type == "Test data portal" assert optional_ds.update_method == UpdateMethodEnum.NO_UPDATES assert optional_ds.readme_url == "https://example.com/readme" @@ -149,4 +147,3 @@ async def test_submit_data_source( "Test record format", "Test record format 2" ] - diff --git a/tests/automated/integration/api/url/by_id/delete/test_data_source_url.py b/tests/automated/integration/api/url/by_id/delete/test_data_source_url.py index d551118b..f8c611a3 100644 --- a/tests/automated/integration/api/url/by_id/delete/test_data_source_url.py +++ b/tests/automated/integration/api/url/by_id/delete/test_data_source_url.py @@ -93,7 +93,6 @@ async def _setup( agency_supplied=False, agency_originated=True, agency_aggregation=AgencyAggregationEnum.LOCALITY, - agency_described_not_in_database="ReadOnly Agency Not In DB", update_method=UpdateMethodEnum.NO_UPDATES, readme_url="https://read-only-readme.com", originating_entity="ReadOnly Agency Originating", diff --git a/tests/automated/integration/readonly/api/data_sources/test_get.py b/tests/automated/integration/readonly/api/data_sources/test_get.py index c23d2177..60369d42 100644 --- a/tests/automated/integration/readonly/api/data_sources/test_get.py +++ b/tests/automated/integration/readonly/api/data_sources/test_get.py @@ -42,7 +42,6 @@ async def test_get(readonly_helper: ReadOnlyTestHelper): agency_supplied=False, agency_originated=True, agency_aggregation=AgencyAggregationEnum.LOCALITY, - agency_described_not_in_database="ReadOnly Agency Not In DB", update_method=UpdateMethodEnum.NO_UPDATES, readme_url="https://read-only-readme.com", originating_entity="ReadOnly Agency Originating", diff --git a/tests/automated/integration/readonly/setup/data_source.py b/tests/automated/integration/readonly/setup/data_source.py index d5984c06..a86f69ea 100644 --- a/tests/automated/integration/readonly/setup/data_source.py +++ b/tests/automated/integration/readonly/setup/data_source.py @@ -49,7 +49,6 @@ async def add_maximal_data_source( agency_supplied=False, agency_originated=True, agency_aggregation=AgencyAggregationEnum.LOCALITY, - agency_described_not_in_database="ReadOnly Agency Not In DB", update_method=UpdateMethodEnum.NO_UPDATES, readme_url="https://read-only-readme.com", originating_entity="ReadOnly Agency Originating", diff --git a/tests/automated/integration/tasks/scheduled/impl/sync_to_ds/data_source/test_add.py b/tests/automated/integration/tasks/scheduled/impl/sync_to_ds/data_source/test_add.py index 2e57e042..d8f2e412 100644 --- a/tests/automated/integration/tasks/scheduled/impl/sync_to_ds/data_source/test_add.py +++ b/tests/automated/integration/tasks/scheduled/impl/sync_to_ds/data_source/test_add.py @@ -69,7 +69,6 @@ async def test_add( assert content.detail_level is None assert content.agency_supplied is None assert content.agency_originated is None - assert content.agency_described_not_in_database is None assert content.update_method is None assert content.readme_url is None assert content.originating_entity is None