Remove agency_described_not_in_database from data source metadata

.gitignore

@@ -10,3 +10,5 @@ runs/
 model.safetensors
 training_args.bin
 /temp/
+.devcontainer/.env.secrets
+.devcontainer/

alembic/versions/2026_02_26_2006-94e2b850fb30_remove_agency_described_not_in_database_.py

@@ -0,0 +1,32 @@
+"""remove agency_described_not_in_database from url optional metadata
+
+Revision ID: 94e2b850fb30
+Revises: 1fb2286a016c
+Create Date: 2026-02-26 20:06:49.423584
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '94e2b850fb30'
+down_revision: Union[str, None] = '1fb2286a016c'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.drop_column(
+        'url_optional_data_source_metadata',
+        'agency_described_not_in_database',
+    )
+
+
+def downgrade() -> None:
+    op.add_column(
+        'url_optional_data_source_metadata',
+        sa.Column('agency_described_not_in_database', sa.Text(), nullable=True),
+    )

src/api/endpoints/data_source/_shared/build.py

@@ -30,7 +30,6 @@ def build_data_source_get_query() -> Select:
             URLOptionalDataSourceMetadata.coverage_end,
             URLOptionalDataSourceMetadata.agency_supplied,
             URLOptionalDataSourceMetadata.agency_aggregation,
-            URLOptionalDataSourceMetadata.agency_described_not_in_database,
             URLOptionalDataSourceMetadata.agency_originated,
             URLOptionalDataSourceMetadata.update_method,
             URLOptionalDataSourceMetadata.readme_url,
@@ -63,4 +62,4 @@ def build_data_source_get_query() -> Select:
         .options(
             selectinload(URL.confirmed_agencies),
         )
-    )
+    )

src/api/endpoints/data_source/_shared/process.py

@@ -32,7 +32,6 @@ def process_data_source_get_mapping(
         agency_supplied=mapping[URLOptionalDataSourceMetadata.agency_supplied],
         agency_aggregation=mapping[URLOptionalDataSourceMetadata.agency_aggregation],
         agency_originated=mapping[URLOptionalDataSourceMetadata.agency_originated],
-        agency_described_not_in_database=mapping[URLOptionalDataSourceMetadata.agency_described_not_in_database],
         update_method=mapping[URLOptionalDataSourceMetadata.update_method],
         readme_url=mapping[URLOptionalDataSourceMetadata.readme_url],
         originating_entity=mapping[URLOptionalDataSourceMetadata.originating_entity],
@@ -41,4 +40,4 @@ def process_data_source_get_mapping(
         submission_notes=mapping[URLOptionalDataSourceMetadata.submission_notes],
         access_notes=mapping[URLOptionalDataSourceMetadata.access_notes],
         access_types=mapping[URLOptionalDataSourceMetadata.access_types] or []
-    )
+    )

src/api/endpoints/data_source/by_id/put/query.py

@@ -70,8 +70,6 @@ async def run(self, session: AsyncSession) -> None:
             value_dict["agency_originated"] = self.request.agency_originated
         if self.request.agency_aggregation is not None:
             value_dict["agency_aggregation"] = self.request.agency_aggregation
-        if self.request.agency_described_not_in_database is not None:
-            value_dict["agency_described_not_in_database"] = self.request.agency_described_not_in_database
         if self.request.update_method is not None:
             value_dict["update_method"] = self.request.update_method
         if self.request.readme_url is not None:
@@ -120,4 +118,3 @@ async def run(self, session: AsyncSession) -> None:
 
             await session.execute(statement)
 
-

src/api/endpoints/data_source/by_id/put/request.py

@@ -27,7 +27,6 @@ class DataSourcePutRequest(BaseModel):
     agency_supplied: bool | None = None
     agency_originated: bool | None = None
     agency_aggregation: AgencyAggregationEnum | None = None
-    agency_described_not_in_database: str | None = None
     update_method: UpdateMethodEnum | None = None
     readme_url: str | None = None
     originating_entity: str | None = None
@@ -47,7 +46,6 @@ def optional_data_source_metadata_not_none(self) -> bool:
             self.agency_supplied is not None or
             self.agency_originated is not None or
             self.agency_aggregation is not None or
-            self.agency_described_not_in_database is not None or
             self.update_method is not None or
             self.readme_url is not None or
             self.originating_entity is not None or
@@ -56,4 +54,4 @@ def optional_data_source_metadata_not_none(self) -> bool:
             self.submission_notes is not None or
             self.access_notes is not None or
             self.access_types is not None
-        )
+        )

src/api/endpoints/data_source/get/response.py

@@ -29,7 +29,6 @@ class DataSourceGetResponse(BaseModel):
     agency_supplied: bool | None = None
     agency_originated: bool | None = None
     agency_aggregation: AgencyAggregationEnum | None = None
-    agency_described_not_in_database: str | None = None
     update_method: UpdateMethodEnum | None = None
     readme_url: str | None = None
     originating_entity: str | None = None

src/api/endpoints/submit/data_source/queries/core.py

@@ -129,7 +129,6 @@ async def run(
             agency_supplied=self.request.agency_supplied,
             agency_originated=self.request.agency_originated,
             agency_aggregation=self.request.agency_aggregation,
-            agency_described_not_in_database=self.request.agency_described_not_in_database,
             data_portal_type=self.request.data_portal_type,
             update_method=self.request.update_method,
             readme_url=self.request.readme_url,
@@ -146,4 +145,4 @@ async def run(
 
         return SubmitDataSourceURLProposalResponse(
             url_id=url_id,
-        )
+        )

src/api/endpoints/submit/data_source/request.py

@@ -20,7 +20,6 @@ class DataSourceSubmissionRequest(RequestBase):
     agency_supplied: bool | None = None
     agency_originated: bool | None = None
     agency_aggregation: AgencyAggregationEnum | None = None
-    agency_described_not_in_database: str | None = None
     data_portal_type: str | None = None
     update_method: UpdateMethodEnum | None = None
     readme_url: str | None = None

src/db/models/impl/url/optional_ds_metadata/sqlalchemy.py

@@ -26,7 +26,6 @@ class URLOptionalDataSourceMetadata(
     agency_supplied = Column(Boolean, nullable=True)
     agency_originated = Column(Boolean, nullable=True)
     agency_aggregation: Mapped[AgencyAggregationEnum] = enum_column(AgencyAggregationEnum, name="agency_aggregation_enum")
-    agency_described_not_in_database = Column(String, nullable=True)
     update_method: Mapped[UpdateMethodEnum] = enum_column(UpdateMethodEnum, name="update_method_enum")
     readme_url = Column(String, nullable=True)
     originating_entity = Column(String, nullable=True)

src/external/pdap/impl/sync/data_sources/_shared/content.py

@@ -28,7 +28,6 @@ class DataSourceSyncContentModel(BaseModel):
     agency_supplied: bool | None = None
     agency_originated: bool | None = None
     agency_aggregation: AgencyAggregationEnum | None = None
-    agency_described_not_in_database: str | None = None
     update_method: UpdateMethodEnum | None = None
     readme_url: str | None = None
     originating_entity: str | None = None

tests/automated/integration/api/data_sources/test_put.py

@@ -39,7 +39,6 @@ async def test_put(
             agency_supplied=False,
             agency_originated=True,
             agency_aggregation=AgencyAggregationEnum.LOCALITY,
-            agency_described_not_in_database="Modified Agency Not In DB",
             update_method=UpdateMethodEnum.OVERWRITE,
             readme_url="https://modified-readme.com",
             originating_entity="Modified Originating Entity",
@@ -78,7 +77,6 @@ async def test_put(
     assert optional_metadata.agency_supplied == False
     assert optional_metadata.agency_originated == True
     assert optional_metadata.agency_aggregation == AgencyAggregationEnum.LOCALITY
-    assert optional_metadata.agency_described_not_in_database == "Modified Agency Not In DB"
     assert optional_metadata.update_method == UpdateMethodEnum.OVERWRITE
     assert optional_metadata.readme_url == "https://modified-readme.com"
     assert optional_metadata.originating_entity == "Modified Originating Entity"

tests/automated/integration/api/submit/data_source/test_core.py

@@ -43,7 +43,6 @@ async def test_submit_data_source(
             agency_supplied=True,
             agency_originated=False,
             agency_aggregation=AgencyAggregationEnum.STATE,
-            agency_described_not_in_database="Test agency described not in database",
             update_method=UpdateMethodEnum.NO_UPDATES,
             readme_url="https://example.com/readme",
             originating_entity="Test Originating Entity",
@@ -131,7 +130,6 @@ async def test_submit_data_source(
     assert optional_ds.agency_supplied
     assert not optional_ds.agency_originated
     assert optional_ds.agency_aggregation == AgencyAggregationEnum.STATE
-    assert optional_ds.agency_described_not_in_database == "Test agency described not in database"
     assert optional_ds.data_portal_type == "Test data portal"
     assert optional_ds.update_method == UpdateMethodEnum.NO_UPDATES
     assert optional_ds.readme_url == "https://example.com/readme"
@@ -149,4 +147,3 @@ async def test_submit_data_source(
         "Test record format",
         "Test record format 2"
     ]
-

tests/automated/integration/api/url/by_id/delete/test_data_source_url.py

@@ -93,7 +93,6 @@ async def _setup(
         agency_supplied=False,
         agency_originated=True,
         agency_aggregation=AgencyAggregationEnum.LOCALITY,
-        agency_described_not_in_database="ReadOnly Agency Not In DB",
         update_method=UpdateMethodEnum.NO_UPDATES,
         readme_url="https://read-only-readme.com",
         originating_entity="ReadOnly Agency Originating",

tests/automated/integration/readonly/api/data_sources/test_get.py

@@ -42,7 +42,6 @@ async def test_get(readonly_helper: ReadOnlyTestHelper):
             agency_supplied=False,
             agency_originated=True,
             agency_aggregation=AgencyAggregationEnum.LOCALITY,
-            agency_described_not_in_database="ReadOnly Agency Not In DB",
             update_method=UpdateMethodEnum.NO_UPDATES,
             readme_url="https://read-only-readme.com",
             originating_entity="ReadOnly Agency Originating",

tests/automated/integration/readonly/setup/data_source.py

@@ -49,7 +49,6 @@ async def add_maximal_data_source(
         agency_supplied=False,
         agency_originated=True,
         agency_aggregation=AgencyAggregationEnum.LOCALITY,
-        agency_described_not_in_database="ReadOnly Agency Not In DB",
         update_method=UpdateMethodEnum.NO_UPDATES,
         readme_url="https://read-only-readme.com",
         originating_entity="ReadOnly Agency Originating",

tests/automated/integration/tasks/scheduled/impl/sync_to_ds/data_source/test_add.py

@@ -69,7 +69,6 @@ async def test_add(
     assert content.detail_level is None
     assert content.agency_supplied is None
     assert content.agency_originated is None
-    assert content.agency_described_not_in_database is None
     assert content.update_method is None
     assert content.readme_url is None
     assert content.originating_entity is None