Add descriptions to data source submissions

src/api/endpoints/submit/data_source/models/response/duplicate.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+
+from src.collectors.enums import URLStatus
+from src.db.models.impl.flag.url_validated.enums import URLType
+
+
+class SubmitDataSourceURLDuplicateSubmissionResponse(BaseModel):
+    message: str
+    url_id: int
+    url_type: URLType | None
+    url_status: URLStatus

src/api/endpoints/submit/data_source/query.py → src/api/endpoints/submit/data_source/queries/core.py

@@ -1,9 +1,10 @@
 from typing import Any
 
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from src.api.endpoints.submit.data_source.models.response.standard import SubmitDataSourceURLProposalResponse
 from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest
-from src.api.endpoints.submit.data_source.response import SubmitDataSourceURLProposalResponse
 from src.collectors.enums import URLStatus
 from src.core.enums import BatchStatus
 from src.db.models.impl.batch.sqlalchemy import Batch
@@ -26,21 +27,27 @@
         super().__init__()
         self.request = request
 
-    async def run(self, session: AsyncSession) -> Any:
+    async def run(
+        self,
+        session: AsyncSession
+    ) -> SubmitDataSourceURLProposalResponse:
         full_url = FullURL(full_url=self.request.source_url)
 
+        # Begin by attempting to submit the full URL
         url = URL(
             url=full_url.id_form,
             scheme=full_url.scheme,
             trailing_slash=full_url.has_trailing_slash,
             name=self.request.name,
+            description=self.request.description,
             status=URLStatus.OK,
             source=URLSource.MANUAL,
         )
 
         session.add(url)
         await session.flush()
 
+        # Standard Path
         url_id: int = url.id
 
         # Add Batch

src/api/endpoints/submit/data_source/queries/duplicate.py

@@ -0,0 +1,58 @@
+from http import HTTPStatus
+
+from fastapi import HTTPException
+from sqlalchemy import select, RowMapping
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.api.endpoints.submit.data_source.models.response.duplicate import \
+    SubmitDataSourceURLDuplicateSubmissionResponse
+from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated
+from src.db.models.impl.url.core.sqlalchemy import URL
+from src.db.queries.base.builder import QueryBuilderBase
+
+
+class GetDataSourceDuplicateQueryBuilder(QueryBuilderBase):
+
+    def __init__(
+        self,
+        url: str
+    ):
+        super().__init__()
+        self.url = url
+
+    async def run(self, session: AsyncSession) -> None:
+        """
+        Raises:
+            HTTPException including details on the duplicate result.
+        """
+
+        query = (
+            select(
+                URL.id,
+                URL.status,
+                FlagURLValidated.type
+            )
+            .outerjoin(
+                FlagURLValidated,
+                FlagURLValidated.url_id == URL.id
+            )
+            .where(
+                URL.url == self.url
+            )
+        )
+        mapping: RowMapping = await self.sh.mapping(
+            query=query,
+            session=session
+        )
+
+        model = SubmitDataSourceURLDuplicateSubmissionResponse(
+            message="Duplicate URL found",
+            url_id=mapping[URL.id],
+            url_status=mapping[URL.status],
+            url_type=mapping[FlagURLValidated.type]
+        )
+        raise HTTPException(
+            detail=model.model_dump(mode='json'),
+            status_code=HTTPStatus.CONFLICT
+        )
+

src/api/endpoints/submit/data_source/request.py

@@ -11,6 +11,7 @@ class DataSourceSubmissionRequest(RequestBase):
     name: str
     record_type: RecordType
     source_url: str
+    description: str | None = None
 
     # Optional URL DS Metadata
     coverage_start: date | None = None

src/api/endpoints/submit/data_source/wrapper.py

@@ -1,8 +1,10 @@
 from fastapi import HTTPException
 
-from src.api.endpoints.submit.data_source.query import SubmitDataSourceURLProposalQueryBuilder
+from src.api.endpoints.submit.data_source.models.response.standard import SubmitDataSourceURLProposalResponse
+from src.api.endpoints.submit.data_source.queries.core import SubmitDataSourceURLProposalQueryBuilder
+
+from src.api.endpoints.submit.data_source.queries.duplicate import GetDataSourceDuplicateQueryBuilder
 from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest
-from src.api.endpoints.submit.data_source.response import SubmitDataSourceURLProposalResponse
 from src.db.client.async_ import AsyncDatabaseClient
 from src.db.queries.urls_exist.model import URLExistsResult
 from src.db.queries.urls_exist.query import URLsExistInDBQueryBuilder
@@ -21,15 +23,18 @@ async def submit_data_source_url_proposal(
             detail="Invalid URL"
         )
 
+    full_url = FullURL(request.source_url)
+
     url_exists_results: URLExistsResult = (await adb_client.run_query_builder(
         URLsExistInDBQueryBuilder(
-            full_urls=[FullURL(request.source_url)]
+            full_urls=[full_url]
         )
     ))[0]
     if url_exists_results.exists:
-        raise HTTPException(
-            status_code=400,
-            detail="URL already exists in database."
+        await adb_client.run_query_builder(
+            GetDataSourceDuplicateQueryBuilder(
+                url=full_url.id_form
+            )
         )
 
     return await adb_client.run_query_builder(

src/api/endpoints/submit/routes.py

@@ -1,8 +1,13 @@
 from fastapi import APIRouter, Depends
 
 from src.api.dependencies import get_async_core
-from src.api.endpoints.submit.data_source.query import SubmitDataSourceURLProposalQueryBuilder
+
+from src.api.endpoints.submit.data_source.models.response.duplicate import \
+    SubmitDataSourceURLDuplicateSubmissionResponse
+from src.api.endpoints.submit.data_source.models.response.standard import SubmitDataSourceURLProposalResponse
+from src.api.endpoints.submit.data_source.queries.core import SubmitDataSourceURLProposalQueryBuilder
 from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest
+from src.api.endpoints.submit.data_source.wrapper import submit_data_source_url_proposal
 from src.api.endpoints.submit.url.models.request import URLSubmissionRequest
 from src.api.endpoints.submit.url.models.response import URLSubmissionResponse
 from src.api.endpoints.submit.url.queries.core import SubmitURLQueryBuilder
@@ -12,7 +17,9 @@
 
 submit_router = APIRouter(prefix="/submit", tags=["submit"])
 
-@submit_router.post("/url")
+@submit_router.post(
+    "/url"
+)
 async def submit_url(
     request: URLSubmissionRequest,
     access_info: AccessInfo = Depends(get_access_info),
@@ -25,13 +32,20 @@
         )
     )
 
-@submit_router.post("/data-source")
+@submit_router.post(
+    "/data-source",
+    response_model=SubmitDataSourceURLProposalResponse,
+    responses={
+        409: {
+            "model": SubmitDataSourceURLDuplicateSubmissionResponse
+        }
+    }
+)
 async def submit_data_source(
     request: DataSourceSubmissionRequest,
     async_core: AsyncCore = Depends(get_async_core),
 ):
-    return await async_core.adb_client.run_query_builder(
-        SubmitDataSourceURLProposalQueryBuilder(
-            request=request,
-        )
+    return await submit_data_source_url_proposal(
+        request=request,
+        adb_client=async_core.adb_client
     )

src/api/endpoints/submit/url/models/request.py

@@ -9,4 +9,5 @@
     record_type: RecordType | None = None
     name: str | None = None
     location_id: int | None = None
-    agency_id: int | None = None
+    agency_id: int | None = None
+    description: str | None = None

src/api/endpoints/submit/url/queries/core.py

@@ -62,6 +62,7 @@ async def run(self, session: AsyncSession) -> URLSubmissionResponse:
             scheme=url_and_scheme.scheme,
             source=URLSource.MANUAL,
             status=URLStatus.OK,
+            description=self.request.description,
             trailing_slash=url_and_scheme.url.endswith('/'),
         )
         session.add(url_insert)

tests/automated/integration/api/submit/data_source/test_core.py

@@ -32,6 +32,7 @@ async def test_submit_data_source(
         json=DataSourceSubmissionRequest(
             source_url="https://example.com/",
             name="Example name",
+            description="Example description",
             record_type=RecordType.COMPLAINTS_AND_MISCONDUCT,
             coverage_start=date(year=2025, month=8, day=9),
             coverage_end=date(year=2025, month=8, day=10),
@@ -74,6 +75,7 @@ async def test_submit_data_source(
     assert url.trailing_slash == True
     assert url.source == URLSource.MANUAL
     assert url.status == URLStatus.OK
+    assert url.description == "Example description"
 
     # Check for Batch
     batch: Batch = await adb_client.one_or_none_model(Batch)

tests/automated/integration/api/submit/data_source/test_duplicate.py

@@ -0,0 +1,38 @@
+import pytest
+from fastapi import HTTPException
+
+from src.api.endpoints.submit.data_source.models.response.duplicate import SubmitDataSourceURLDuplicateSubmissionResponse
+from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest
+from src.collectors.enums import URLStatus
+from src.core.enums import RecordType
+from src.db.dtos.url.mapping_.simple import SimpleURLMapping
+from src.db.models.impl.flag.url_validated.enums import URLType
+from tests.helpers.api_test_helper import APITestHelper
+from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo
+
+
+@pytest.mark.asyncio
+async def test_submit_data_source_duplicate(
+    api_test_helper: APITestHelper,
+    test_agency_id: int,
+    pittsburgh_locality: LocalityCreationInfo,
+    test_url_data_source_mapping: SimpleURLMapping
+):
+
+    ath = api_test_helper
+    try:
+        ath.request_validator.post_v3(
+            url="submit/data-source",
+            json=DataSourceSubmissionRequest(
+                source_url=test_url_data_source_mapping.url,
+                name="Test Name",
+                record_type=RecordType.RECORDS_REQUEST_INFO
+            ).model_dump(mode='json')
+        )
+    except HTTPException as e:
+        response = e.detail['detail']
+        model = SubmitDataSourceURLDuplicateSubmissionResponse(**response)
+        assert model.url_id == test_url_data_source_mapping.url_id
+        assert model.url_type == URLType.DATA_SOURCE
+        assert model.url_status == URLStatus.OK
+        assert model.message == "Duplicate URL found"

tests/automated/integration/api/submit/test_url_maximal.py

@@ -32,6 +32,7 @@ async def test_maximal(
         request=URLSubmissionRequest(
             url="www.example.com",
             record_type=RecordType.INCARCERATION_RECORDS,
+            description="Example description",
             name="Example URL",
             location_id=pittsburgh_locality.location_id,
             agency_id=agency_id,
@@ -48,6 +49,7 @@ async def test_maximal(
     url: URL = urls[0]
     assert url.id == url_id
     assert url.url == "www.example.com"
+    assert url.description == "Example description"
 
     links: list[LinkUserSubmittedURL] = await adb_client.get_all(LinkUserSubmittedURL)
     assert len(links) == 1

tests/automated/integration/conftest.py

@@ -12,6 +12,7 @@
 from src.core.logger import AsyncCoreLogger
 from src.db.client.async_ import AsyncDatabaseClient
 from src.db.client.sync import DatabaseClient
+from src.db.dtos.url.mapping_.simple import SimpleURLMapping
 from src.db.models.impl.flag.url_validated.enums import URLType
 from src.security.dtos.access_info import AccessInfo
 from src.security.enums import Permissions
@@ -217,6 +218,21 @@
     )
     return url_id
 
+@pytest_asyncio.fixture
+async def test_url_data_source_mapping(
+    db_data_creator: DBDataCreator,
+    test_agency_id: int
+) -> SimpleURLMapping:
+    url_mapping: SimpleURLMapping = (await db_data_creator.create_validated_urls(
+        record_type=RecordType.CRIME_STATISTICS,
+        validation_type=URLType.DATA_SOURCE,
+    ))[0]
+    await db_data_creator.link_urls_to_agencies(
+        url_ids=[url_mapping.url_id],
+        agency_ids=[test_agency_id]
+    )
+    return url_mapping
+
 @pytest_asyncio.fixture
 async def test_url_meta_url_id(
     db_data_creator: DBDataCreator,