From 7e6e4c79ce531275d226eb50fa8f423d47f27fd1 Mon Sep 17 00:00:00 2001 From: Max Chis Date: Tue, 30 Sep 2025 09:28:00 -0400 Subject: [PATCH 1/2] Begin draft --- src/api/endpoints/submit/__init__.py | 0 src/api/endpoints/submit/routes.py | 18 +++++ src/api/endpoints/submit/urls/__init__.py | 0 src/api/endpoints/submit/urls/enums.py | 14 ++++ .../endpoints/submit/urls/models/__init__.py | 0 .../endpoints/submit/urls/models/request.py | 5 ++ .../endpoints/submit/urls/models/response.py | 15 +++++ .../endpoints/submit/urls/queries/__init__.py | 0 .../submit/urls/queries/clean/__init__.py | 0 .../submit/urls/queries/clean/core.py | 5 ++ .../submit/urls/queries/clean/response.py | 5 ++ .../endpoints/submit/urls/queries/convert.py | 20 ++++++ src/api/endpoints/submit/urls/queries/core.py | 66 +++++++++++++++++++ .../urls/queries/deduplicate/__init__.py | 0 .../submit/urls/queries/deduplicate/core.py | 39 +++++++++++ .../urls/queries/deduplicate/response.py | 6 ++ .../submit/urls/queries/validate/__init__.py | 0 .../submit/urls/queries/validate/core.py | 5 ++ .../submit/urls/queries/validate/response.py | 6 ++ 19 files changed, 204 insertions(+) create mode 100644 src/api/endpoints/submit/__init__.py create mode 100644 src/api/endpoints/submit/routes.py create mode 100644 src/api/endpoints/submit/urls/__init__.py create mode 100644 src/api/endpoints/submit/urls/enums.py create mode 100644 src/api/endpoints/submit/urls/models/__init__.py create mode 100644 src/api/endpoints/submit/urls/models/request.py create mode 100644 src/api/endpoints/submit/urls/models/response.py create mode 100644 src/api/endpoints/submit/urls/queries/__init__.py create mode 100644 src/api/endpoints/submit/urls/queries/clean/__init__.py create mode 100644 src/api/endpoints/submit/urls/queries/clean/core.py create mode 100644 src/api/endpoints/submit/urls/queries/clean/response.py create mode 100644 src/api/endpoints/submit/urls/queries/convert.py create mode 100644 src/api/endpoints/submit/urls/queries/core.py create mode 100644 src/api/endpoints/submit/urls/queries/deduplicate/__init__.py create mode 100644 src/api/endpoints/submit/urls/queries/deduplicate/core.py create mode 100644 src/api/endpoints/submit/urls/queries/deduplicate/response.py create mode 100644 src/api/endpoints/submit/urls/queries/validate/__init__.py create mode 100644 src/api/endpoints/submit/urls/queries/validate/core.py create mode 100644 src/api/endpoints/submit/urls/queries/validate/response.py diff --git a/src/api/endpoints/submit/__init__.py b/src/api/endpoints/submit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/routes.py b/src/api/endpoints/submit/routes.py new file mode 100644 index 00000000..e342120f --- /dev/null +++ b/src/api/endpoints/submit/routes.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter, Depends + +from src.api.dependencies import get_async_core +from src.api.endpoints.submit.urls.models.request import URLSubmissionRequest +from src.api.endpoints.submit.urls.models.response import URLBatchSubmissionResponse +from src.core.core import AsyncCore +from src.security.dtos.access_info import AccessInfo +from src.security.manager import get_access_info + +submit_router = APIRouter(prefix="/submit", tags=["submit"]) + +@submit_router.post("/urls") +async def submit_urls( + urls: URLSubmissionRequest, + access_info: AccessInfo = Depends(get_access_info), + async_core: AsyncCore = Depends(get_async_core), +) -> URLBatchSubmissionResponse: + raise NotImplementedError \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/__init__.py b/src/api/endpoints/submit/urls/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/urls/enums.py b/src/api/endpoints/submit/urls/enums.py new file mode 100644 index 00000000..ca86c5df --- /dev/null +++ b/src/api/endpoints/submit/urls/enums.py @@ -0,0 +1,14 @@ +from enum import Enum + + +class URLBatchSubmissionStatus(Enum): + ALL_ACCEPTED = "all_accepted" + PARTIALLY_ACCEPTED = "partially_accepted" + ALL_REJECTED = "all_rejected" + +class URLSubmissionStatus(Enum): + ACCEPTED_AS_IS = "accepted_as_is" + ACCEPTED_WITH_CLEANING = "accepted_with_cleaning" + BATCH_DUPLICATE = "batch_duplicate" + DATABASE_DUPLICATE = "database_duplicate" + INVALID = "invalid" \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/models/__init__.py b/src/api/endpoints/submit/urls/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/urls/models/request.py b/src/api/endpoints/submit/urls/models/request.py new file mode 100644 index 00000000..073b7e1e --- /dev/null +++ b/src/api/endpoints/submit/urls/models/request.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + + +class URLSubmissionRequest(BaseModel): + urls: list[str] \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/models/response.py b/src/api/endpoints/submit/urls/models/response.py new file mode 100644 index 00000000..5239f2d0 --- /dev/null +++ b/src/api/endpoints/submit/urls/models/response.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel + +from src.api.endpoints.submit.urls.enums import URLBatchSubmissionStatus, URLSubmissionStatus + + +class URLSubmissionResponse(BaseModel): + url_original: str + url_cleaned: str | None = None + status: URLSubmissionStatus + url_id: int | None = None + +class URLBatchSubmissionResponse(BaseModel): + status: URLBatchSubmissionStatus + batch_id: int | None + urls: list[URLSubmissionResponse] \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/__init__.py b/src/api/endpoints/submit/urls/queries/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/urls/queries/clean/__init__.py b/src/api/endpoints/submit/urls/queries/clean/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/urls/queries/clean/core.py b/src/api/endpoints/submit/urls/queries/clean/core.py new file mode 100644 index 00000000..31bc19c0 --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/clean/core.py @@ -0,0 +1,5 @@ +from src.api.endpoints.submit.urls.queries.clean.response import CleanURLResponse + + +def clean_urls(urls: list[str]) -> list[CleanURLResponse]: + raise NotImplementedError \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/clean/response.py b/src/api/endpoints/submit/urls/queries/clean/response.py new file mode 100644 index 00000000..58e98d1a --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/clean/response.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + +class CleanURLResponse(BaseModel): + url_original: str + url_cleaned: str diff --git a/src/api/endpoints/submit/urls/queries/convert.py b/src/api/endpoints/submit/urls/queries/convert.py new file mode 100644 index 00000000..3461a3ee --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/convert.py @@ -0,0 +1,20 @@ +from src.api.endpoints.submit.urls.enums import URLSubmissionStatus +from src.api.endpoints.submit.urls.models.response import URLSubmissionResponse + + +def convert_invalid_urls_to_url_response( + urls: list[str] +) -> list[URLSubmissionResponse]: + return [ + URLSubmissionResponse( + url_original=url, + status=URLSubmissionStatus.INVALID, + ) + for url in urls + ] + +def convert_duplicate_urls_to_url_response( + clean_urls: list[str], + url_clean_original_mapping: dict[str, str] +) -> list[URLSubmissionResponse]: + raise NotImplementedError \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/core.py b/src/api/endpoints/submit/urls/queries/core.py new file mode 100644 index 00000000..4fb6ce7a --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/core.py @@ -0,0 +1,66 @@ +from typing import Any, Counter + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.endpoints.submit.urls.enums import URLSubmissionStatus +from src.api.endpoints.submit.urls.models.response import URLBatchSubmissionResponse, URLSubmissionResponse +from src.api.endpoints.submit.urls.queries.clean.core import clean_urls +from src.api.endpoints.submit.urls.queries.clean.response import CleanURLResponse +from src.api.endpoints.submit.urls.queries.convert import convert_invalid_urls_to_url_response +from src.api.endpoints.submit.urls.queries.deduplicate.core import DeduplicateURLsQueryBuilder +from src.api.endpoints.submit.urls.queries.deduplicate.response import DeduplicateURLResponse +from src.api.endpoints.submit.urls.queries.validate.core import validate_urls +from src.api.endpoints.submit.urls.queries.validate.response import ValidateURLResponse +from src.db.queries.base.builder import QueryBuilderBase + + +class SubmitURLsQueryBuilder(QueryBuilderBase): + + def __init__( + self, + urls: list[str], + ): + super().__init__() + self.urls = urls + + async def run(self, session: AsyncSession) -> URLBatchSubmissionResponse: + url_responses: list[URLSubmissionResponse] = [] + url_clean_original_mapping: dict[str, str] = {} + + # Filter out invalid URLs + validate_response: ValidateURLResponse = validate_urls(self.urls) + invalid_url_responses: list[URLSubmissionResponse] = convert_invalid_urls_to_url_response( + validate_response.invalid_urls + ) + url_responses.extend(invalid_url_responses) + valid_urls: list[str] = validate_response.valid_urls + + # Clean URLs + clean_url_responses: list[CleanURLResponse] = clean_urls(valid_urls) + for clean_url_response in clean_url_responses: + url_clean_original_mapping[clean_url_response.url_cleaned] = \ + clean_url_response.url_original + + # Filter out within-batch duplicates + clean_url_set: set[str] = set() + for clean_url_response in clean_url_responses: + cur = clean_url_response + if cur.url_cleaned in clean_url_set: + url_responses.append( + URLSubmissionResponse( + url_original=cur.url_original, + url_cleaned=cur.url_cleaned, + status=URLSubmissionStatus.BATCH_DUPLICATE, + url_id=None, + ) + ) + else: + clean_url_set.add(cur.url_cleaned) + clean_url_list: list[str] = list(clean_url_set) + + # Filter out within-database duplicates + deduplicate_response: DeduplicateURLResponse = \ + await DeduplicateURLsQueryBuilder(clean_url_list).run(session) + + + # Submit URLs and get URL ids diff --git a/src/api/endpoints/submit/urls/queries/deduplicate/__init__.py b/src/api/endpoints/submit/urls/queries/deduplicate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/urls/queries/deduplicate/core.py b/src/api/endpoints/submit/urls/queries/deduplicate/core.py new file mode 100644 index 00000000..f2c48859 --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/deduplicate/core.py @@ -0,0 +1,39 @@ +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.endpoints.submit.urls.queries.deduplicate.response import DeduplicateURLResponse +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.queries.base.builder import QueryBuilderBase + +from src.db.helpers.session import session_helper as sh + +class DeduplicateURLsQueryBuilder(QueryBuilderBase): + + def __init__(self, urls: list[str]): + super().__init__() + self.urls = urls + + async def run(self, session: AsyncSession) -> DeduplicateURLResponse: + + query = select( + URL.url + ).where( + URL.url.in_(self.urls) + ) + + results: list[str] = await sh.scalars(session, query=query) + results_set: set[str] = set(results) + + new_urls: list[str] = list(set(self.urls) - results_set) + duplicate_urls: list[str] = list(set(self.urls) & results_set) + + return DeduplicateURLResponse( + new_urls=new_urls, + duplicate_urls=duplicate_urls, + ) + + + + diff --git a/src/api/endpoints/submit/urls/queries/deduplicate/response.py b/src/api/endpoints/submit/urls/queries/deduplicate/response.py new file mode 100644 index 00000000..4961b42a --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/deduplicate/response.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class DeduplicateURLResponse(BaseModel): + new_urls: list[str] + duplicate_urls: list[str] \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/validate/__init__.py b/src/api/endpoints/submit/urls/queries/validate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/urls/queries/validate/core.py b/src/api/endpoints/submit/urls/queries/validate/core.py new file mode 100644 index 00000000..8994e609 --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/validate/core.py @@ -0,0 +1,5 @@ +from src.api.endpoints.submit.urls.queries.validate.response import ValidateURLResponse + + +def validate_urls(urls: list[str]) -> ValidateURLResponse: + raise NotImplementedError \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/validate/response.py b/src/api/endpoints/submit/urls/queries/validate/response.py new file mode 100644 index 00000000..e24d3f28 --- /dev/null +++ b/src/api/endpoints/submit/urls/queries/validate/response.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class ValidateURLResponse(BaseModel): + valid_urls: list[str] + invalid_urls: list[str] \ No newline at end of file From 269985c415dd5a3d87d54544108333edec00c701 Mon Sep 17 00:00:00 2001 From: maxachis Date: Tue, 30 Sep 2025 12:10:19 -0400 Subject: [PATCH 2/2] Add `submit/url` endpoint --- ...26ad8_add_link_user_submitted_url_table.py | 34 +++++ src/api/endpoints/submit/routes.py | 20 ++- .../submit/{urls => url}/__init__.py | 0 .../endpoints/submit/{urls => url}/enums.py | 7 - .../submit/{urls => url}/models/__init__.py | 0 .../endpoints/submit/url/models/request.py | 11 ++ .../endpoints/submit/url/models/response.py | 18 +++ .../submit/{urls => url}/queries/__init__.py | 0 .../endpoints/submit/url/queries/convert.py | 21 +++ src/api/endpoints/submit/url/queries/core.py | 128 ++++++++++++++++++ .../endpoints/submit/url/queries/dedupe.py | 28 ++++ .../endpoints/submit/urls/models/request.py | 5 - .../endpoints/submit/urls/models/response.py | 15 -- .../submit/urls/queries/clean/core.py | 5 - .../submit/urls/queries/clean/response.py | 5 - .../endpoints/submit/urls/queries/convert.py | 20 --- src/api/endpoints/submit/urls/queries/core.py | 66 --------- .../submit/urls/queries/deduplicate/core.py | 39 ------ .../urls/queries/deduplicate/response.py | 6 - .../submit/urls/queries/validate/__init__.py | 0 .../submit/urls/queries/validate/core.py | 5 - .../submit/urls/queries/validate/response.py | 6 - src/api/main.py | 4 +- .../users_submitted_url}/__init__.py | 0 .../users_submitted_url/sqlalchemy.py | 19 +++ src/db/utils/validate.py | 16 ++- .../api/_helpers/RequestValidator.py | 14 +- .../integration/api/submit}/__init__.py | 0 .../integration/api/submit/test_duplicate.py | 24 ++++ .../integration/api/submit/test_invalid.py | 16 +++ .../api/submit/test_needs_cleaning.py | 37 +++++ .../api/submit/test_url_maximal.py | 85 ++++++++++++ .../api/submit/test_url_minimal.py | 37 +++++ 33 files changed, 502 insertions(+), 189 deletions(-) create mode 100644 alembic/versions/2025_09_30_1046-84a3de626ad8_add_link_user_submitted_url_table.py rename src/api/endpoints/submit/{urls => url}/__init__.py (100%) rename src/api/endpoints/submit/{urls => url}/enums.py (52%) rename src/api/endpoints/submit/{urls => url}/models/__init__.py (100%) create mode 100644 src/api/endpoints/submit/url/models/request.py create mode 100644 src/api/endpoints/submit/url/models/response.py rename src/api/endpoints/submit/{urls => url}/queries/__init__.py (100%) create mode 100644 src/api/endpoints/submit/url/queries/convert.py create mode 100644 src/api/endpoints/submit/url/queries/core.py create mode 100644 src/api/endpoints/submit/url/queries/dedupe.py delete mode 100644 src/api/endpoints/submit/urls/models/request.py delete mode 100644 src/api/endpoints/submit/urls/models/response.py delete mode 100644 src/api/endpoints/submit/urls/queries/clean/core.py delete mode 100644 src/api/endpoints/submit/urls/queries/clean/response.py delete mode 100644 src/api/endpoints/submit/urls/queries/convert.py delete mode 100644 src/api/endpoints/submit/urls/queries/core.py delete mode 100644 src/api/endpoints/submit/urls/queries/deduplicate/core.py delete mode 100644 src/api/endpoints/submit/urls/queries/deduplicate/response.py delete mode 100644 src/api/endpoints/submit/urls/queries/validate/__init__.py delete mode 100644 src/api/endpoints/submit/urls/queries/validate/core.py delete mode 100644 src/api/endpoints/submit/urls/queries/validate/response.py rename src/{api/endpoints/submit/urls/queries/clean => db/models/impl/link/user_suggestion_not_found/users_submitted_url}/__init__.py (100%) create mode 100644 src/db/models/impl/link/user_suggestion_not_found/users_submitted_url/sqlalchemy.py rename {src/api/endpoints/submit/urls/queries/deduplicate => tests/automated/integration/api/submit}/__init__.py (100%) create mode 100644 tests/automated/integration/api/submit/test_duplicate.py create mode 100644 tests/automated/integration/api/submit/test_invalid.py create mode 100644 tests/automated/integration/api/submit/test_needs_cleaning.py create mode 100644 tests/automated/integration/api/submit/test_url_maximal.py create mode 100644 tests/automated/integration/api/submit/test_url_minimal.py diff --git a/alembic/versions/2025_09_30_1046-84a3de626ad8_add_link_user_submitted_url_table.py b/alembic/versions/2025_09_30_1046-84a3de626ad8_add_link_user_submitted_url_table.py new file mode 100644 index 00000000..73735610 --- /dev/null +++ b/alembic/versions/2025_09_30_1046-84a3de626ad8_add_link_user_submitted_url_table.py @@ -0,0 +1,34 @@ +"""Add link user submitted URL table + +Revision ID: 84a3de626ad8 +Revises: 5be534715a01 +Create Date: 2025-09-30 10:46:16.552174 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from src.util.alembic_helpers import url_id_column, user_id_column, created_at_column + +# revision identifiers, used by Alembic. +revision: str = '84a3de626ad8' +down_revision: Union[str, None] = '5be534715a01' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "link_user_submitted_urls", + url_id_column(), + user_id_column(), + created_at_column(), + sa.PrimaryKeyConstraint("url_id", "user_id"), + sa.UniqueConstraint("url_id") + ) + + +def downgrade() -> None: + pass diff --git a/src/api/endpoints/submit/routes.py b/src/api/endpoints/submit/routes.py index e342120f..d91d1821 100644 --- a/src/api/endpoints/submit/routes.py +++ b/src/api/endpoints/submit/routes.py @@ -1,18 +1,24 @@ from fastapi import APIRouter, Depends from src.api.dependencies import get_async_core -from src.api.endpoints.submit.urls.models.request import URLSubmissionRequest -from src.api.endpoints.submit.urls.models.response import URLBatchSubmissionResponse +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse +from src.api.endpoints.submit.url.queries.core import SubmitURLQueryBuilder from src.core.core import AsyncCore from src.security.dtos.access_info import AccessInfo from src.security.manager import get_access_info submit_router = APIRouter(prefix="/submit", tags=["submit"]) -@submit_router.post("/urls") -async def submit_urls( - urls: URLSubmissionRequest, +@submit_router.post("/url") +async def submit_url( + request: URLSubmissionRequest, access_info: AccessInfo = Depends(get_access_info), async_core: AsyncCore = Depends(get_async_core), -) -> URLBatchSubmissionResponse: - raise NotImplementedError \ No newline at end of file +) -> URLSubmissionResponse: + return await async_core.adb_client.run_query_builder( + SubmitURLQueryBuilder( + request=request, + user_id=access_info.user_id + ) + ) \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/__init__.py b/src/api/endpoints/submit/url/__init__.py similarity index 100% rename from src/api/endpoints/submit/urls/__init__.py rename to src/api/endpoints/submit/url/__init__.py diff --git a/src/api/endpoints/submit/urls/enums.py b/src/api/endpoints/submit/url/enums.py similarity index 52% rename from src/api/endpoints/submit/urls/enums.py rename to src/api/endpoints/submit/url/enums.py index ca86c5df..08802072 100644 --- a/src/api/endpoints/submit/urls/enums.py +++ b/src/api/endpoints/submit/url/enums.py @@ -1,14 +1,7 @@ from enum import Enum - -class URLBatchSubmissionStatus(Enum): - ALL_ACCEPTED = "all_accepted" - PARTIALLY_ACCEPTED = "partially_accepted" - ALL_REJECTED = "all_rejected" - class URLSubmissionStatus(Enum): ACCEPTED_AS_IS = "accepted_as_is" ACCEPTED_WITH_CLEANING = "accepted_with_cleaning" - BATCH_DUPLICATE = "batch_duplicate" DATABASE_DUPLICATE = "database_duplicate" INVALID = "invalid" \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/models/__init__.py b/src/api/endpoints/submit/url/models/__init__.py similarity index 100% rename from src/api/endpoints/submit/urls/models/__init__.py rename to src/api/endpoints/submit/url/models/__init__.py diff --git a/src/api/endpoints/submit/url/models/request.py b/src/api/endpoints/submit/url/models/request.py new file mode 100644 index 00000000..5b52d761 --- /dev/null +++ b/src/api/endpoints/submit/url/models/request.py @@ -0,0 +1,11 @@ +from pydantic import BaseModel + +from src.core.enums import RecordType + + +class URLSubmissionRequest(BaseModel): + url: str + record_type: RecordType | None = None + name: str | None = None + location_id: int | None = None + agency_id: int | None = None \ No newline at end of file diff --git a/src/api/endpoints/submit/url/models/response.py b/src/api/endpoints/submit/url/models/response.py new file mode 100644 index 00000000..f2f8d031 --- /dev/null +++ b/src/api/endpoints/submit/url/models/response.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel, model_validator + +from src.api.endpoints.submit.url.enums import URLSubmissionStatus + + +class URLSubmissionResponse(BaseModel): + url_original: str + url_cleaned: str | None = None + status: URLSubmissionStatus + url_id: int | None = None + + @model_validator(mode="after") + def validate_url_id_if_accepted(self): + if self.status in [URLSubmissionStatus.ACCEPTED_AS_IS, URLSubmissionStatus.ACCEPTED_WITH_CLEANING]: + if self.url_id is None: + raise ValueError("url_id is required for accepted urls") + return self + diff --git a/src/api/endpoints/submit/urls/queries/__init__.py b/src/api/endpoints/submit/url/queries/__init__.py similarity index 100% rename from src/api/endpoints/submit/urls/queries/__init__.py rename to src/api/endpoints/submit/url/queries/__init__.py diff --git a/src/api/endpoints/submit/url/queries/convert.py b/src/api/endpoints/submit/url/queries/convert.py new file mode 100644 index 00000000..90a32566 --- /dev/null +++ b/src/api/endpoints/submit/url/queries/convert.py @@ -0,0 +1,21 @@ +from src.api.endpoints.submit.url.enums import URLSubmissionStatus +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse + + +def convert_invalid_url_to_url_response( + url: str +) -> URLSubmissionResponse: + return URLSubmissionResponse( + url_original=url, + status=URLSubmissionStatus.INVALID, + ) + +def convert_duplicate_urls_to_url_response( + clean_url: str, + original_url: str +) -> URLSubmissionResponse: + return URLSubmissionResponse( + url_original=original_url, + url_cleaned=clean_url, + status=URLSubmissionStatus.DATABASE_DUPLICATE, + ) diff --git a/src/api/endpoints/submit/url/queries/core.py b/src/api/endpoints/submit/url/queries/core.py new file mode 100644 index 00000000..081b5456 --- /dev/null +++ b/src/api/endpoints/submit/url/queries/core.py @@ -0,0 +1,128 @@ + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.endpoints.submit.url.enums import URLSubmissionStatus +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse +from src.api.endpoints.submit.url.queries.convert import convert_invalid_url_to_url_response, \ + convert_duplicate_urls_to_url_response +from src.api.endpoints.submit.url.queries.dedupe import DeduplicateURLQueryBuilder +from src.collectors.enums import URLStatus +from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL +from src.db.models.impl.url.core.enums import URLSource +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.models.impl.url.suggestion.agency.user import UserUrlAgencySuggestion +from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion +from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource +from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion +from src.db.queries.base.builder import QueryBuilderBase +from src.db.utils.validate import is_valid_url +from src.util.clean import clean_url + + +class SubmitURLQueryBuilder(QueryBuilderBase): + + def __init__( + self, + request: URLSubmissionRequest, + user_id: int + ): + super().__init__() + self.request = request + self.user_id = user_id + + async def run(self, session: AsyncSession) -> URLSubmissionResponse: + url_original: str = self.request.url + + # Filter out invalid URLs + valid: bool = is_valid_url(url_original) + if not valid: + return convert_invalid_url_to_url_response(url_original) + + # Clean URLs + url_clean: str = clean_url(url_original) + + # Check if duplicate + is_duplicate: bool = await DeduplicateURLQueryBuilder(url=url_clean).run(session) + if is_duplicate: + return convert_duplicate_urls_to_url_response( + clean_url=url_clean, + original_url=url_original + ) + + # Submit URLs and get URL id + + # Add URL + url_insert = URL( + url=url_clean, + source=URLSource.MANUAL, + status=URLStatus.OK, + ) + session.add(url_insert) + await session.flush() + + # Add Link + link = LinkUserSubmittedURL( + url_id=url_insert.id, + user_id=self.user_id, + ) + session.add(link) + + # Add record type as suggestion if exists + if self.request.record_type is not None: + rec_sugg = UserRecordTypeSuggestion( + user_id=self.user_id, + url_id=url_insert.id, + record_type=self.request.record_type.value + ) + session.add(rec_sugg) + + # Add name as suggestion if exists + if self.request.name is not None: + name_sugg = URLNameSuggestion( + url_id=url_insert.id, + suggestion=self.request.name, + source=NameSuggestionSource.USER + ) + session.add(name_sugg) + await session.flush() + + link_name_sugg = LinkUserNameSuggestion( + suggestion_id=name_sugg.id, + user_id=self.user_id + ) + session.add(link_name_sugg) + + + + # Add location ID as suggestion if exists + if self.request.location_id is not None: + loc_sugg = UserLocationSuggestion( + user_id=self.user_id, + url_id=url_insert.id, + location_id=self.request.location_id + ) + session.add(loc_sugg) + + # Add agency ID as suggestion if exists + if self.request.agency_id is not None: + agen_sugg = UserUrlAgencySuggestion( + user_id=self.user_id, + url_id=url_insert.id, + agency_id=self.request.agency_id + ) + session.add(agen_sugg) + + if url_clean == url_original: + status = URLSubmissionStatus.ACCEPTED_AS_IS + else: + status = URLSubmissionStatus.ACCEPTED_WITH_CLEANING + + return URLSubmissionResponse( + url_original=url_original, + url_cleaned=url_clean, + status=status, + url_id=url_insert.id, + ) diff --git a/src/api/endpoints/submit/url/queries/dedupe.py b/src/api/endpoints/submit/url/queries/dedupe.py new file mode 100644 index 00000000..43c92edd --- /dev/null +++ b/src/api/endpoints/submit/url/queries/dedupe.py @@ -0,0 +1,28 @@ +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.db.helpers.session import session_helper as sh +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.queries.base.builder import QueryBuilderBase + + +class DeduplicateURLQueryBuilder(QueryBuilderBase): + + def __init__(self, url: str): + super().__init__() + self.url = url + + async def run(self, session: AsyncSession) -> bool: + + query = select( + URL.url + ).where( + URL.url == self.url + ) + + return await sh.has_results(session, query=query) + + + + + diff --git a/src/api/endpoints/submit/urls/models/request.py b/src/api/endpoints/submit/urls/models/request.py deleted file mode 100644 index 073b7e1e..00000000 --- a/src/api/endpoints/submit/urls/models/request.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class URLSubmissionRequest(BaseModel): - urls: list[str] \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/models/response.py b/src/api/endpoints/submit/urls/models/response.py deleted file mode 100644 index 5239f2d0..00000000 --- a/src/api/endpoints/submit/urls/models/response.py +++ /dev/null @@ -1,15 +0,0 @@ -from pydantic import BaseModel - -from src.api.endpoints.submit.urls.enums import URLBatchSubmissionStatus, URLSubmissionStatus - - -class URLSubmissionResponse(BaseModel): - url_original: str - url_cleaned: str | None = None - status: URLSubmissionStatus - url_id: int | None = None - -class URLBatchSubmissionResponse(BaseModel): - status: URLBatchSubmissionStatus - batch_id: int | None - urls: list[URLSubmissionResponse] \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/clean/core.py b/src/api/endpoints/submit/urls/queries/clean/core.py deleted file mode 100644 index 31bc19c0..00000000 --- a/src/api/endpoints/submit/urls/queries/clean/core.py +++ /dev/null @@ -1,5 +0,0 @@ -from src.api.endpoints.submit.urls.queries.clean.response import CleanURLResponse - - -def clean_urls(urls: list[str]) -> list[CleanURLResponse]: - raise NotImplementedError \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/clean/response.py b/src/api/endpoints/submit/urls/queries/clean/response.py deleted file mode 100644 index 58e98d1a..00000000 --- a/src/api/endpoints/submit/urls/queries/clean/response.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - -class CleanURLResponse(BaseModel): - url_original: str - url_cleaned: str diff --git a/src/api/endpoints/submit/urls/queries/convert.py b/src/api/endpoints/submit/urls/queries/convert.py deleted file mode 100644 index 3461a3ee..00000000 --- a/src/api/endpoints/submit/urls/queries/convert.py +++ /dev/null @@ -1,20 +0,0 @@ -from src.api.endpoints.submit.urls.enums import URLSubmissionStatus -from src.api.endpoints.submit.urls.models.response import URLSubmissionResponse - - -def convert_invalid_urls_to_url_response( - urls: list[str] -) -> list[URLSubmissionResponse]: - return [ - URLSubmissionResponse( - url_original=url, - status=URLSubmissionStatus.INVALID, - ) - for url in urls - ] - -def convert_duplicate_urls_to_url_response( - clean_urls: list[str], - url_clean_original_mapping: dict[str, str] -) -> list[URLSubmissionResponse]: - raise NotImplementedError \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/core.py b/src/api/endpoints/submit/urls/queries/core.py deleted file mode 100644 index 4fb6ce7a..00000000 --- a/src/api/endpoints/submit/urls/queries/core.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import Any, Counter - -from sqlalchemy.ext.asyncio import AsyncSession - -from src.api.endpoints.submit.urls.enums import URLSubmissionStatus -from src.api.endpoints.submit.urls.models.response import URLBatchSubmissionResponse, URLSubmissionResponse -from src.api.endpoints.submit.urls.queries.clean.core import clean_urls -from src.api.endpoints.submit.urls.queries.clean.response import CleanURLResponse -from src.api.endpoints.submit.urls.queries.convert import convert_invalid_urls_to_url_response -from src.api.endpoints.submit.urls.queries.deduplicate.core import DeduplicateURLsQueryBuilder -from src.api.endpoints.submit.urls.queries.deduplicate.response import DeduplicateURLResponse -from src.api.endpoints.submit.urls.queries.validate.core import validate_urls -from src.api.endpoints.submit.urls.queries.validate.response import ValidateURLResponse -from src.db.queries.base.builder import QueryBuilderBase - - -class SubmitURLsQueryBuilder(QueryBuilderBase): - - def __init__( - self, - urls: list[str], - ): - super().__init__() - self.urls = urls - - async def run(self, session: AsyncSession) -> URLBatchSubmissionResponse: - url_responses: list[URLSubmissionResponse] = [] - url_clean_original_mapping: dict[str, str] = {} - - # Filter out invalid URLs - validate_response: ValidateURLResponse = validate_urls(self.urls) - invalid_url_responses: list[URLSubmissionResponse] = convert_invalid_urls_to_url_response( - validate_response.invalid_urls - ) - url_responses.extend(invalid_url_responses) - valid_urls: list[str] = validate_response.valid_urls - - # Clean URLs - clean_url_responses: list[CleanURLResponse] = clean_urls(valid_urls) - for clean_url_response in clean_url_responses: - url_clean_original_mapping[clean_url_response.url_cleaned] = \ - clean_url_response.url_original - - # Filter out within-batch duplicates - clean_url_set: set[str] = set() - for clean_url_response in clean_url_responses: - cur = clean_url_response - if cur.url_cleaned in clean_url_set: - url_responses.append( - URLSubmissionResponse( - url_original=cur.url_original, - url_cleaned=cur.url_cleaned, - status=URLSubmissionStatus.BATCH_DUPLICATE, - url_id=None, - ) - ) - else: - clean_url_set.add(cur.url_cleaned) - clean_url_list: list[str] = list(clean_url_set) - - # Filter out within-database duplicates - deduplicate_response: DeduplicateURLResponse = \ - await DeduplicateURLsQueryBuilder(clean_url_list).run(session) - - - # Submit URLs and get URL ids diff --git a/src/api/endpoints/submit/urls/queries/deduplicate/core.py b/src/api/endpoints/submit/urls/queries/deduplicate/core.py deleted file mode 100644 index f2c48859..00000000 --- a/src/api/endpoints/submit/urls/queries/deduplicate/core.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Any - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from src.api.endpoints.submit.urls.queries.deduplicate.response import DeduplicateURLResponse -from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.queries.base.builder import QueryBuilderBase - -from src.db.helpers.session import session_helper as sh - -class DeduplicateURLsQueryBuilder(QueryBuilderBase): - - def __init__(self, urls: list[str]): - super().__init__() - self.urls = urls - - async def run(self, session: AsyncSession) -> DeduplicateURLResponse: - - query = select( - URL.url - ).where( - URL.url.in_(self.urls) - ) - - results: list[str] = await sh.scalars(session, query=query) - results_set: set[str] = set(results) - - new_urls: list[str] = list(set(self.urls) - results_set) - duplicate_urls: list[str] = list(set(self.urls) & results_set) - - return DeduplicateURLResponse( - new_urls=new_urls, - duplicate_urls=duplicate_urls, - ) - - - - diff --git a/src/api/endpoints/submit/urls/queries/deduplicate/response.py b/src/api/endpoints/submit/urls/queries/deduplicate/response.py deleted file mode 100644 index 4961b42a..00000000 --- a/src/api/endpoints/submit/urls/queries/deduplicate/response.py +++ /dev/null @@ -1,6 +0,0 @@ -from pydantic import BaseModel - - -class DeduplicateURLResponse(BaseModel): - new_urls: list[str] - duplicate_urls: list[str] \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/validate/__init__.py b/src/api/endpoints/submit/urls/queries/validate/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/api/endpoints/submit/urls/queries/validate/core.py b/src/api/endpoints/submit/urls/queries/validate/core.py deleted file mode 100644 index 8994e609..00000000 --- a/src/api/endpoints/submit/urls/queries/validate/core.py +++ /dev/null @@ -1,5 +0,0 @@ -from src.api.endpoints.submit.urls.queries.validate.response import ValidateURLResponse - - -def validate_urls(urls: list[str]) -> ValidateURLResponse: - raise NotImplementedError \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/validate/response.py b/src/api/endpoints/submit/urls/queries/validate/response.py deleted file mode 100644 index e24d3f28..00000000 --- a/src/api/endpoints/submit/urls/queries/validate/response.py +++ /dev/null @@ -1,6 +0,0 @@ -from pydantic import BaseModel - - -class ValidateURLResponse(BaseModel): - valid_urls: list[str] - invalid_urls: list[str] \ No newline at end of file diff --git a/src/api/main.py b/src/api/main.py index ddf44a5b..1eb0a22b 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -14,6 +14,7 @@ from src.api.endpoints.review.routes import review_router from src.api.endpoints.root import root_router from src.api.endpoints.search.routes import search_router +from src.api.endpoints.submit.routes import submit_router from src.api.endpoints.task.routes import task_router from src.api.endpoints.url.routes import url_router from src.collectors.impl.muckrock.api_interface.core import MuckrockAPIInterface @@ -175,7 +176,8 @@ async def redirect_docs(): task_router, review_router, search_router, - metrics_router + metrics_router, + submit_router ] for router in routers: diff --git a/src/api/endpoints/submit/urls/queries/clean/__init__.py b/src/db/models/impl/link/user_suggestion_not_found/users_submitted_url/__init__.py similarity index 100% rename from src/api/endpoints/submit/urls/queries/clean/__init__.py rename to src/db/models/impl/link/user_suggestion_not_found/users_submitted_url/__init__.py diff --git a/src/db/models/impl/link/user_suggestion_not_found/users_submitted_url/sqlalchemy.py b/src/db/models/impl/link/user_suggestion_not_found/users_submitted_url/sqlalchemy.py new file mode 100644 index 00000000..7407c016 --- /dev/null +++ b/src/db/models/impl/link/user_suggestion_not_found/users_submitted_url/sqlalchemy.py @@ -0,0 +1,19 @@ +from sqlalchemy import Column, Integer, PrimaryKeyConstraint, UniqueConstraint +from sqlalchemy.orm import Mapped + +from src.db.models.mixins import URLDependentMixin, CreatedAtMixin +from src.db.models.templates_.base import Base + + +class LinkUserSubmittedURL( + Base, + URLDependentMixin, + CreatedAtMixin, +): + __tablename__ = "link_user_submitted_url" + __table_args__ = ( + PrimaryKeyConstraint("url_id", "user_id"), + UniqueConstraint("url_id"), + ) + + user_id: Mapped[int] \ No newline at end of file diff --git a/src/db/utils/validate.py b/src/db/utils/validate.py index 077b7752..4837e12c 100644 --- a/src/db/utils/validate.py +++ b/src/db/utils/validate.py @@ -1,4 +1,5 @@ from typing import Protocol +from urllib.parse import urlparse from pydantic import BaseModel @@ -10,4 +11,17 @@ def validate_has_protocol(obj: object, protocol: type[Protocol]): def validate_all_models_of_same_type(objects: list[object]): first_model = objects[0] if not all(isinstance(model, type(first_model)) for model in objects): - raise TypeError("Models must be of the same type") \ No newline at end of file + raise TypeError("Models must be of the same type") + +def is_valid_url(url: str) -> bool: + try: + result = urlparse(url) + # If scheme is missing, `netloc` will be empty, so we check path too + if result.scheme in ("http", "https") and result.netloc: + return True + if not result.scheme and result.path: + # no scheme, treat path as potential domain + return "." in result.path + return False + except ValueError: + return False diff --git a/tests/automated/integration/api/_helpers/RequestValidator.py b/tests/automated/integration/api/_helpers/RequestValidator.py index d7cfbf42..6847da1b 100644 --- a/tests/automated/integration/api/_helpers/RequestValidator.py +++ b/tests/automated/integration/api/_helpers/RequestValidator.py @@ -26,6 +26,8 @@ from src.api.endpoints.review.next.dto import GetNextURLForFinalReviewOuterResponse from src.api.endpoints.review.reject.dto import FinalReviewRejectionInfo from src.api.endpoints.search.dtos.response import SearchURLResponse +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse from src.api.endpoints.task.by_id.dto import TaskInfo from src.api.endpoints.task.dtos.get.task_status import GetTaskStatusResponseInfo from src.api.endpoints.task.dtos.get.tasks import GetTasksResponse @@ -419,4 +421,14 @@ async def get_url_screenshot(self, url_id: int) -> Response: return self.client.get( url=f"/url/{url_id}/screenshot", headers={"Authorization": f"Bearer token"} - ) \ No newline at end of file + ) + + async def submit_url( + self, + request: URLSubmissionRequest + ) -> URLSubmissionResponse: + response: dict = self.post_v2( + url="/submit/url", + json=request.model_dump(mode='json') + ) + return URLSubmissionResponse(**response) \ No newline at end of file diff --git a/src/api/endpoints/submit/urls/queries/deduplicate/__init__.py b/tests/automated/integration/api/submit/__init__.py similarity index 100% rename from src/api/endpoints/submit/urls/queries/deduplicate/__init__.py rename to tests/automated/integration/api/submit/__init__.py diff --git a/tests/automated/integration/api/submit/test_duplicate.py b/tests/automated/integration/api/submit/test_duplicate.py new file mode 100644 index 00000000..c1ccfd29 --- /dev/null +++ b/tests/automated/integration/api/submit/test_duplicate.py @@ -0,0 +1,24 @@ +import pytest + +from src.api.endpoints.submit.url.enums import URLSubmissionStatus +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse +from src.db.dtos.url.mapping import URLMapping +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.data_creator.core import DBDataCreator + + +@pytest.mark.asyncio +async def test_duplicate( + api_test_helper: APITestHelper, + db_data_creator: DBDataCreator +): + url_mapping: URLMapping = (await db_data_creator.create_urls(count=1))[0] + + response: URLSubmissionResponse = await api_test_helper.request_validator.submit_url( + request=URLSubmissionRequest( + url=url_mapping.url + ) + ) + assert response.status == URLSubmissionStatus.DATABASE_DUPLICATE + assert response.url_id is None \ No newline at end of file diff --git a/tests/automated/integration/api/submit/test_invalid.py b/tests/automated/integration/api/submit/test_invalid.py new file mode 100644 index 00000000..a5ae27e7 --- /dev/null +++ b/tests/automated/integration/api/submit/test_invalid.py @@ -0,0 +1,16 @@ +import pytest + +from src.api.endpoints.submit.url.enums import URLSubmissionStatus +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse +from tests.helpers.api_test_helper import APITestHelper + + +@pytest.mark.asyncio +async def test_invalid(api_test_helper: APITestHelper): + response: URLSubmissionResponse = await api_test_helper.request_validator.submit_url( + request=URLSubmissionRequest( + url="invalid_url" + ) + ) + assert response.status == URLSubmissionStatus.INVALID \ No newline at end of file diff --git a/tests/automated/integration/api/submit/test_needs_cleaning.py b/tests/automated/integration/api/submit/test_needs_cleaning.py new file mode 100644 index 00000000..85c2f112 --- /dev/null +++ b/tests/automated/integration/api/submit/test_needs_cleaning.py @@ -0,0 +1,37 @@ +import pytest + +from src.api.endpoints.submit.url.enums import URLSubmissionStatus +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL +from src.db.models.impl.url.core.sqlalchemy import URL +from tests.helpers.api_test_helper import APITestHelper + + +@pytest.mark.asyncio +async def test_needs_cleaning( + api_test_helper: APITestHelper, + adb_client_test: AsyncDatabaseClient +): + response: URLSubmissionResponse = await api_test_helper.request_validator.submit_url( + request=URLSubmissionRequest( + url="www.example.com#fragment" + ) + ) + + assert response.status == URLSubmissionStatus.ACCEPTED_WITH_CLEANING + assert response.url_id is not None + url_id: int = response.url_id + + adb_client: AsyncDatabaseClient = adb_client_test + urls: list[URL] = await adb_client.get_all(URL) + assert len(urls) == 1 + url: URL = urls[0] + assert url.id == url_id + assert url.url == "www.example.com" + + links: list[LinkUserSubmittedURL] = await adb_client.get_all(LinkUserSubmittedURL) + assert len(links) == 1 + link: LinkUserSubmittedURL = links[0] + assert link.url_id == url_id \ No newline at end of file diff --git a/tests/automated/integration/api/submit/test_url_maximal.py b/tests/automated/integration/api/submit/test_url_maximal.py new file mode 100644 index 00000000..8d1930f5 --- /dev/null +++ b/tests/automated/integration/api/submit/test_url_maximal.py @@ -0,0 +1,85 @@ +import pytest + +from src.api.endpoints.submit.url.enums import URLSubmissionStatus +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse +from src.core.enums import RecordType +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.models.impl.url.suggestion.agency.user import UserUrlAgencySuggestion +from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion +from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource +from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.data_creator.core import DBDataCreator +from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo + + +@pytest.mark.asyncio +async def test_maximal( + api_test_helper: APITestHelper, + adb_client_test: AsyncDatabaseClient, + db_data_creator: DBDataCreator, + pittsburgh_locality: LocalityCreationInfo +): + + agency_id: int = await db_data_creator.agency() + + response: URLSubmissionResponse = await api_test_helper.request_validator.submit_url( + request=URLSubmissionRequest( + url="www.example.com", + record_type=RecordType.INCARCERATION_RECORDS, + name="Example URL", + location_id=pittsburgh_locality.location_id, + agency_id=agency_id, + ) + ) + + assert response.status == URLSubmissionStatus.ACCEPTED_AS_IS + assert response.url_id is not None + url_id: int = response.url_id + + adb_client: AsyncDatabaseClient = adb_client_test + urls: list[URL] = await adb_client.get_all(URL) + assert len(urls) == 1 + url: URL = urls[0] + assert url.id == url_id + assert url.url == "www.example.com" + + links: list[LinkUserSubmittedURL] = await adb_client.get_all(LinkUserSubmittedURL) + assert len(links) == 1 + link: LinkUserSubmittedURL = links[0] + assert link.url_id == url_id + + agen_suggs: list[UserUrlAgencySuggestion] = await adb_client.get_all(UserUrlAgencySuggestion) + assert len(agen_suggs) == 1 + agen_sugg: UserUrlAgencySuggestion = agen_suggs[0] + assert agen_sugg.url_id == url_id + assert agen_sugg.agency_id == agency_id + + loc_suggs: list[UserLocationSuggestion] = await adb_client.get_all(UserLocationSuggestion) + assert len(loc_suggs) == 1 + loc_sugg: UserLocationSuggestion = loc_suggs[0] + assert loc_sugg.url_id == url_id + assert loc_sugg.location_id == pittsburgh_locality.location_id + + name_sugg: list[URLNameSuggestion] = await adb_client.get_all(URLNameSuggestion) + assert len(name_sugg) == 1 + name_sugg: URLNameSuggestion = name_sugg[0] + assert name_sugg.url_id == url_id + assert name_sugg.suggestion == "Example URL" + assert name_sugg.source == NameSuggestionSource.USER + + name_link_suggs: list[LinkUserNameSuggestion] = await adb_client.get_all(LinkUserNameSuggestion) + assert len(name_link_suggs) == 1 + name_link_sugg: LinkUserNameSuggestion = name_link_suggs[0] + assert name_link_sugg.suggestion_id == name_sugg.id + + rec_suggs: list[UserRecordTypeSuggestion] = await adb_client.get_all(UserRecordTypeSuggestion) + assert len(rec_suggs) == 1 + rec_sugg: UserRecordTypeSuggestion = rec_suggs[0] + assert rec_sugg.url_id == url_id + assert rec_sugg.record_type == RecordType.INCARCERATION_RECORDS.value diff --git a/tests/automated/integration/api/submit/test_url_minimal.py b/tests/automated/integration/api/submit/test_url_minimal.py new file mode 100644 index 00000000..f1f078f6 --- /dev/null +++ b/tests/automated/integration/api/submit/test_url_minimal.py @@ -0,0 +1,37 @@ +import pytest + +from src.api.endpoints.submit.url.enums import URLSubmissionStatus +from src.api.endpoints.submit.url.models.request import URLSubmissionRequest +from src.api.endpoints.submit.url.models.response import URLSubmissionResponse +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL +from src.db.models.impl.url.core.sqlalchemy import URL +from tests.helpers.api_test_helper import APITestHelper + + +@pytest.mark.asyncio +async def test_minimal( + api_test_helper: APITestHelper, + adb_client_test: AsyncDatabaseClient +): + response: URLSubmissionResponse = await api_test_helper.request_validator.submit_url( + request=URLSubmissionRequest( + url="www.example.com" + ) + ) + + assert response.status == URLSubmissionStatus.ACCEPTED_AS_IS + assert response.url_id is not None + url_id: int = response.url_id + + adb_client: AsyncDatabaseClient = adb_client_test + urls: list[URL] = await adb_client.get_all(URL) + assert len(urls) == 1 + url: URL = urls[0] + assert url.id == url_id + assert url.url == "www.example.com" + + links: list[LinkUserSubmittedURL] = await adb_client.get_all(LinkUserSubmittedURL) + assert len(links) == 1 + link: LinkUserSubmittedURL = links[0] + assert link.url_id == url_id \ No newline at end of file