mc_455_submit_urls

alembic/versions/2025_09_30_1046-84a3de626ad8_add_link_user_submitted_url_table.py

@@ -0,0 +1,34 @@
+"""Add link user submitted URL table
+
+Revision ID: 84a3de626ad8
+Revises: 5be534715a01
+Create Date: 2025-09-30 10:46:16.552174
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from src.util.alembic_helpers import url_id_column, user_id_column, created_at_column
+
+# revision identifiers, used by Alembic.
+revision: str = '84a3de626ad8'
+down_revision: Union[str, None] = '5be534715a01'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "link_user_submitted_urls",
+        url_id_column(),
+        user_id_column(),
+        created_at_column(),
+        sa.PrimaryKeyConstraint("url_id", "user_id"),
+        sa.UniqueConstraint("url_id")
+    )
+
+
+def downgrade() -> None:
+    pass

src/api/endpoints/submit/routes.py

@@ -0,0 +1,24 @@
+from fastapi import APIRouter, Depends
+
+from src.api.dependencies import get_async_core
+from src.api.endpoints.submit.url.models.request import URLSubmissionRequest
+from src.api.endpoints.submit.url.models.response import URLSubmissionResponse
+from src.api.endpoints.submit.url.queries.core import SubmitURLQueryBuilder
+from src.core.core import AsyncCore
+from src.security.dtos.access_info import AccessInfo
+from src.security.manager import get_access_info
+
+submit_router = APIRouter(prefix="/submit", tags=["submit"])
+
+@submit_router.post("/url")
+async def submit_url(
+    request: URLSubmissionRequest,
+    access_info: AccessInfo = Depends(get_access_info),
+    async_core: AsyncCore = Depends(get_async_core),
+) -> URLSubmissionResponse:
+    return await async_core.adb_client.run_query_builder(
+        SubmitURLQueryBuilder(
+            request=request,
+            user_id=access_info.user_id
+        )
+    )

src/api/endpoints/submit/url/enums.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+class URLSubmissionStatus(Enum):
+    ACCEPTED_AS_IS = "accepted_as_is"
+    ACCEPTED_WITH_CLEANING = "accepted_with_cleaning"
+    DATABASE_DUPLICATE = "database_duplicate"
+    INVALID = "invalid"

src/api/endpoints/submit/url/models/request.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+
+from src.core.enums import RecordType
+
+
+class URLSubmissionRequest(BaseModel):
+    url: str
+    record_type: RecordType | None = None
+    name: str | None = None
+    location_id: int | None = None
+    agency_id: int | None = None

src/api/endpoints/submit/url/models/response.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, model_validator
+
+from src.api.endpoints.submit.url.enums import URLSubmissionStatus
+
+
+class URLSubmissionResponse(BaseModel):
+    url_original: str
+    url_cleaned: str | None = None
+    status: URLSubmissionStatus
+    url_id: int | None = None
+
+    @model_validator(mode="after")
+    def validate_url_id_if_accepted(self):
+        if self.status in [URLSubmissionStatus.ACCEPTED_AS_IS, URLSubmissionStatus.ACCEPTED_WITH_CLEANING]:
+            if self.url_id is None:
+                raise ValueError("url_id is required for accepted urls")
+        return self
+

src/api/endpoints/submit/url/queries/convert.py

@@ -0,0 +1,21 @@
+from src.api.endpoints.submit.url.enums import URLSubmissionStatus
+from src.api.endpoints.submit.url.models.response import URLSubmissionResponse
+
+
+def convert_invalid_url_to_url_response(
+    url: str
+) -> URLSubmissionResponse:
+    return URLSubmissionResponse(
+        url_original=url,
+        status=URLSubmissionStatus.INVALID,
+    )
+
+def convert_duplicate_urls_to_url_response(
+    clean_url: str,
+    original_url: str
+) -> URLSubmissionResponse:
+    return URLSubmissionResponse(
+        url_original=original_url,
+        url_cleaned=clean_url,
+        status=URLSubmissionStatus.DATABASE_DUPLICATE,
+    )

src/api/endpoints/submit/url/queries/core.py

@@ -0,0 +1,128 @@
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.api.endpoints.submit.url.enums import URLSubmissionStatus
+from src.api.endpoints.submit.url.models.request import URLSubmissionRequest
+from src.api.endpoints.submit.url.models.response import URLSubmissionResponse
+from src.api.endpoints.submit.url.queries.convert import convert_invalid_url_to_url_response, \
+    convert_duplicate_urls_to_url_response
+from src.api.endpoints.submit.url.queries.dedupe import DeduplicateURLQueryBuilder
+from src.collectors.enums import URLStatus
+from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion
+from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL
+from src.db.models.impl.url.core.enums import URLSource
+from src.db.models.impl.url.core.sqlalchemy import URL
+from src.db.models.impl.url.suggestion.agency.user import UserUrlAgencySuggestion
+from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion
+from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource
+from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion
+from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion
+from src.db.queries.base.builder import QueryBuilderBase
+from src.db.utils.validate import is_valid_url
+from src.util.clean import clean_url
+
+
+class SubmitURLQueryBuilder(QueryBuilderBase):
+
+    def __init__(
+        self,
+        request: URLSubmissionRequest,
+        user_id: int
+    ):
+        super().__init__()
+        self.request = request
+        self.user_id = user_id
+
+    async def run(self, session: AsyncSession) -> URLSubmissionResponse:
+        url_original: str = self.request.url
+
+        # Filter out invalid URLs
+        valid: bool = is_valid_url(url_original)
+        if not valid:
+            return convert_invalid_url_to_url_response(url_original)
+
+        # Clean URLs
+        url_clean: str = clean_url(url_original)
+
+        # Check if duplicate
+        is_duplicate: bool = await DeduplicateURLQueryBuilder(url=url_clean).run(session)
+        if is_duplicate:
+            return convert_duplicate_urls_to_url_response(
+                clean_url=url_clean,
+                original_url=url_original
+            )
+
+        # Submit URLs and get URL id
+
+        # Add URL
+        url_insert = URL(
+            url=url_clean,
+            source=URLSource.MANUAL,
+            status=URLStatus.OK,
+        )
+        session.add(url_insert)
+        await session.flush()
+
+        # Add Link
+        link = LinkUserSubmittedURL(
+            url_id=url_insert.id,
+            user_id=self.user_id,
+        )
+        session.add(link)
+
+        # Add record type as suggestion if exists
+        if self.request.record_type is not None:
+            rec_sugg = UserRecordTypeSuggestion(
+                user_id=self.user_id,
+                url_id=url_insert.id,
+                record_type=self.request.record_type.value
+            )
+            session.add(rec_sugg)
+
+        # Add name as suggestion if exists
+        if self.request.name is not None:
+            name_sugg = URLNameSuggestion(
+                url_id=url_insert.id,
+                suggestion=self.request.name,
+                source=NameSuggestionSource.USER
+            )
+            session.add(name_sugg)
+            await session.flush()
+
+            link_name_sugg = LinkUserNameSuggestion(
+                suggestion_id=name_sugg.id,
+                user_id=self.user_id
+            )
+            session.add(link_name_sugg)
+
+
+
+        # Add location ID as suggestion if exists
+        if self.request.location_id is not None:
+            loc_sugg = UserLocationSuggestion(
+                user_id=self.user_id,
+                url_id=url_insert.id,
+                location_id=self.request.location_id
+            )
+            session.add(loc_sugg)
+
+        # Add agency ID as suggestion if exists
+        if self.request.agency_id is not None:
+            agen_sugg = UserUrlAgencySuggestion(
+                user_id=self.user_id,
+                url_id=url_insert.id,
+                agency_id=self.request.agency_id
+            )
+            session.add(agen_sugg)
+
+        if url_clean == url_original:
+            status = URLSubmissionStatus.ACCEPTED_AS_IS
+        else:
+            status = URLSubmissionStatus.ACCEPTED_WITH_CLEANING
+
+        return URLSubmissionResponse(
+            url_original=url_original,
+            url_cleaned=url_clean,
+            status=status,
+            url_id=url_insert.id,
+        )

src/api/endpoints/submit/url/queries/dedupe.py

@@ -0,0 +1,28 @@
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.db.helpers.session import session_helper as sh
+from src.db.models.impl.url.core.sqlalchemy import URL
+from src.db.queries.base.builder import QueryBuilderBase
+
+
+class DeduplicateURLQueryBuilder(QueryBuilderBase):
+
+    def __init__(self, url: str):
+        super().__init__()
+        self.url = url
+
+    async def run(self, session: AsyncSession) -> bool:
+
+        query = select(
+            URL.url
+        ).where(
+            URL.url == self.url
+        )
+
+        return await sh.has_results(session, query=query)
+
+
+
+
+

src/api/main.py

@@ -14,6 +14,7 @@
 from src.api.endpoints.review.routes import review_router
 from src.api.endpoints.root import root_router
 from src.api.endpoints.search.routes import search_router
+from src.api.endpoints.submit.routes import submit_router
 from src.api.endpoints.task.routes import task_router
 from src.api.endpoints.url.routes import url_router
 from src.collectors.impl.muckrock.api_interface.core import MuckrockAPIInterface
@@ -175,7 +176,8 @@ async def redirect_docs():
     task_router,
     review_router,
     search_router,
-    metrics_router
+    metrics_router,
+    submit_router
 ]
 
 for router in routers:

src/db/models/impl/link/user_suggestion_not_found/users_submitted_url/sqlalchemy.py

@@ -0,0 +1,19 @@
+from sqlalchemy import Column, Integer, PrimaryKeyConstraint, UniqueConstraint
+from sqlalchemy.orm import Mapped
+
+from src.db.models.mixins import URLDependentMixin, CreatedAtMixin
+from src.db.models.templates_.base import Base
+
+
+class LinkUserSubmittedURL(
+    Base,
+    URLDependentMixin,
+    CreatedAtMixin,
+):
+    __tablename__ = "link_user_submitted_url"
+    __table_args__ = (
+        PrimaryKeyConstraint("url_id", "user_id"),
+        UniqueConstraint("url_id"),
+    )
+
+    user_id: Mapped[int]

src/db/utils/validate.py

@@ -1,4 +1,5 @@
 from typing import Protocol
+from urllib.parse import urlparse
 
 from pydantic import BaseModel
 
@@ -10,4 +11,17 @@
 def validate_all_models_of_same_type(objects: list[object]):
     first_model = objects[0]
     if not all(isinstance(model, type(first_model)) for model in objects):
-        raise TypeError("Models must be of the same type")
+        raise TypeError("Models must be of the same type")
+
+def is_valid_url(url: str) -> bool:
+    try:
+        result = urlparse(url)
+        # If scheme is missing, `netloc` will be empty, so we check path too
+        if result.scheme in ("http", "https") and result.netloc:
+            return True
+        if not result.scheme and result.path:
+            # no scheme, treat path as potential domain
+            return "." in result.path
+        return False
+    except ValueError:
+        return False

tests/automated/integration/api/_helpers/RequestValidator.py

@@ -26,6 +26,8 @@
 from src.api.endpoints.review.next.dto import GetNextURLForFinalReviewOuterResponse
 from src.api.endpoints.review.reject.dto import FinalReviewRejectionInfo
 from src.api.endpoints.search.dtos.response import SearchURLResponse
+from src.api.endpoints.submit.url.models.request import URLSubmissionRequest
+from src.api.endpoints.submit.url.models.response import URLSubmissionResponse
 from src.api.endpoints.task.by_id.dto import TaskInfo
 from src.api.endpoints.task.dtos.get.task_status import GetTaskStatusResponseInfo
 from src.api.endpoints.task.dtos.get.tasks import GetTasksResponse
@@ -419,4 +421,14 @@
         return self.client.get(
             url=f"/url/{url_id}/screenshot",
             headers={"Authorization": f"Bearer token"}
-        )
+        )
+
+    async def submit_url(
+        self,
+        request: URLSubmissionRequest
+    ) -> URLSubmissionResponse:
+        response: dict = self.post_v2(
+            url="/submit/url",
+            json=request.model_dump(mode='json')
+        )
+        return URLSubmissionResponse(**response)

tests/automated/integration/api/submit/test_duplicate.py

@@ -0,0 +1,24 @@
+import pytest
+
+from src.api.endpoints.submit.url.enums import URLSubmissionStatus
+from src.api.endpoints.submit.url.models.request import URLSubmissionRequest
+from src.api.endpoints.submit.url.models.response import URLSubmissionResponse
+from src.db.dtos.url.mapping import URLMapping
+from tests.helpers.api_test_helper import APITestHelper
+from tests.helpers.data_creator.core import DBDataCreator
+
+
+@pytest.mark.asyncio
+async def test_duplicate(
+    api_test_helper: APITestHelper,
+    db_data_creator: DBDataCreator
+):
+    url_mapping: URLMapping = (await db_data_creator.create_urls(count=1))[0]
+
+    response: URLSubmissionResponse = await api_test_helper.request_validator.submit_url(
+        request=URLSubmissionRequest(
+            url=url_mapping.url
+        )
+    )
+    assert response.status == URLSubmissionStatus.DATABASE_DUPLICATE
+    assert response.url_id is None