mc_452_missing_locations_agencies

ENV.md

@@ -72,20 +72,21 @@ Note that some tasks/subtasks are themselves enabled by other tasks.
 URL Task Flags are collectively controlled by the `RUN_URL_TASKS_TASK_FLAG` flag.
 
 
-| Flag                                | Description                                                        |
-|-------------------------------------|--------------------------------------------------------------------|
-| `URL_HTML_TASK_FLAG`                | URL HTML scraping task.                                |
-| `URL_RECORD_TYPE_TASK_FLAG`         | Automatically assigns Record Types to URLs.            |
-| `URL_AGENCY_IDENTIFICATION_TASK_FLAG` | Automatically assigns and suggests Agencies for URLs.  |
-| `URL_SUBMIT_APPROVED_TASK_FLAG`     | Submits approved URLs to the Data Sources App.         |
-| `URL_MISC_METADATA_TASK_FLAG`       | Adds misc metadata to URLs.                            |
-| `URL_404_PROBE_TASK_FLAG`           | Probes URLs for 404 errors.                            |
-| `URL_AUTO_RELEVANCE_TASK_FLAG`      | Automatically assigns Relevances to URLs.              |
-| `URL_PROBE_TASK_FLAG`               | Probes URLs for web metadata.                          |
-| `URL_ROOT_URL_TASK_FLAG`            | Extracts and links Root URLs to URLs.                  |
-| `URL_SCREENSHOT_TASK_FLAG`          | Takes screenshots of URLs.                             |
-| `URL_AUTO_VALIDATE_TASK_FLAG`       | Automatically validates URLs.                          |
-| `URL_AUTO_NAME_TASK_FLAG`           | Automatically names URLs.                              |
+| Flag                                | Description                                           |
+|-------------------------------------|-------------------------------------------------------|
+| `URL_HTML_TASK_FLAG`                | URL HTML scraping task.                               |
+| `URL_RECORD_TYPE_TASK_FLAG`         | Automatically assigns Record Types to URLs.           |
+| `URL_AGENCY_IDENTIFICATION_TASK_FLAG` | Automatically assigns and suggests Agencies for URLs. |
+| `URL_SUBMIT_APPROVED_TASK_FLAG`     | Submits approved URLs to the Data Sources App.        |
+| `URL_MISC_METADATA_TASK_FLAG`       | Adds misc metadata to URLs.                           |
+| `URL_404_PROBE_TASK_FLAG`           | Probes URLs for 404 errors.                           |
+| `URL_AUTO_RELEVANCE_TASK_FLAG`      | Automatically assigns Relevances to URLs.             |
+| `URL_PROBE_TASK_FLAG`               | Probes URLs for web metadata.                         |
+| `URL_ROOT_URL_TASK_FLAG`            | Extracts and links Root URLs to URLs.                 |
+| `URL_SCREENSHOT_TASK_FLAG`          | Takes screenshots of URLs.                            |
+| `URL_AUTO_VALIDATE_TASK_FLAG`       | Automatically validates URLs.                         |
+| `URL_AUTO_NAME_TASK_FLAG`           | Automatically names URLs.                             |
+| `URL_SUSPEND_TASK_FLAG`             | Suspends URLs meeting suspension criteria.            |
 
 ### Agency ID Subtasks
 

alembic/versions/2025_09_29_1246-5be534715a01_add_agency_location_not_found_logic.py

@@ -0,0 +1,74 @@
+"""Add Agency/Location Not Found Logic
+
+Revision ID: 5be534715a01
+Revises: 50a710e413f8
+Create Date: 2025-09-29 12:46:27.140173
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from src.util.alembic_helpers import created_at_column, url_id_column, user_id_column
+
+# revision identifiers, used by Alembic.
+revision: str = '5be534715a01'
+down_revision: Union[str, None] = '50a710e413f8'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+def upgrade() -> None:
+    add_link_user_suggestion_agency_not_found_table()
+    add_link_user_suggestion_location_not_found_table()
+    add_flag_url_suspended_table()
+    add_url_suspend_task_type()
+    remove_link_url_new_agency_suggestion_table()
+    remove_new_agency_suggestions_table()
+
+def add_url_suspend_task_type():
+    op.execute(
+        """
+        ALTER TYPE task_type ADD VALUE 'Suspend URLs';
+        """
+    )
+
+def add_link_user_suggestion_agency_not_found_table():
+    op.create_table(
+        "link_user_suggestion_agency_not_found",
+        user_id_column(),
+        url_id_column(),
+        created_at_column(),
+        sa.PrimaryKeyConstraint("user_id", "url_id"),
+    )
+
+
+def add_link_user_suggestion_location_not_found_table():
+    op.create_table(
+        "link_user_suggestion_location_not_found",
+        user_id_column(),
+        url_id_column(),
+        created_at_column(),
+        sa.PrimaryKeyConstraint("user_id", "url_id"),
+    )
+
+
+def add_flag_url_suspended_table():
+    op.create_table(
+        "flag_url_suspended",
+        url_id_column(),
+        created_at_column(),
+        sa.PrimaryKeyConstraint("url_id"),
+    )
+
+
+def remove_link_url_new_agency_suggestion_table():
+    op.drop_table("link_url_new_agency_suggestion")
+
+
+def remove_new_agency_suggestions_table():
+    op.drop_table("new_agency_suggestions")
+
+
+def downgrade() -> None:
+    pass

src/api/endpoints/annotate/all/get/models/agency.py

@@ -0,0 +1,27 @@
+from pydantic import BaseModel, Field
+
+
+class AgencyAnnotationAutoSuggestion(BaseModel):
+    agency_id: int
+    agency_name: str
+    confidence: int = Field(
+        title="The confidence of the location",
+        ge=0,
+        le=100,
+    )
+
+class AgencyAnnotationUserSuggestion(BaseModel):
+    agency_id: int
+    agency_name: str
+    user_count: int
+
+class AgencyAnnotationUserSuggestionOuterInfo(BaseModel):
+    suggestions: list[AgencyAnnotationUserSuggestion]
+    not_found_count: int = Field(
+        title="How many users listed the agency as not found.",
+        ge=0,
+    )
+
+class AgencyAnnotationResponseOuterInfo(BaseModel):
+    user: AgencyAnnotationUserSuggestionOuterInfo
+    auto: list[AgencyAnnotationAutoSuggestion]

src/api/endpoints/annotate/all/get/models/location.py

@@ -23,7 +23,13 @@
         ge=1,
     )
 
+class LocationAnnotationUserSuggestionOuterInfo(BaseModel):
+    suggestions: list[LocationAnnotationUserSuggestion]
+    not_found_count: int = Field(
+        title="How many users listed the location as not found.",
+        ge=0,
+    )
 
 class LocationAnnotationResponseOuterInfo(BaseModel):
-    user: list[LocationAnnotationUserSuggestion]
+    user: LocationAnnotationUserSuggestionOuterInfo
     auto: list[LocationAnnotationAutoSuggestion]

src/api/endpoints/annotate/all/get/models/response.py

@@ -3,6 +3,7 @@
 from pydantic import Field, BaseModel
 
 from src.api.endpoints.annotate.agency.get.dto import GetNextURLForAgencyAgencyInfo
+from src.api.endpoints.annotate.all.get.models.agency import AgencyAnnotationResponseOuterInfo
 from src.api.endpoints.annotate.all.get.models.location import LocationAnnotationResponseOuterInfo
 from src.api.endpoints.annotate.all.get.models.name import NameAnnotationSuggestion
 from src.api.endpoints.annotate.all.get.models.record_type import RecordTypeAnnotationSuggestion
@@ -13,7 +14,7 @@
 
 
 class GetNextURLForAllAnnotationInnerResponse(AnnotationInnerResponseInfoBase):
-    agency_suggestions: list[GetNextURLForAgencyAgencyInfo] | None = Field(
+    agency_suggestions: AgencyAnnotationResponseOuterInfo | None = Field(
         title="The auto-labeler's suggestions for agencies"
     )
     location_suggestions: LocationAnnotationResponseOuterInfo | None = Field(

src/api/endpoints/annotate/all/get/queries/agency/core.py

@@ -0,0 +1,44 @@
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.api.endpoints.annotate.all.get.models.agency import AgencyAnnotationResponseOuterInfo, \
+    AgencyAnnotationUserSuggestionOuterInfo, AgencyAnnotationUserSuggestion, AgencyAnnotationAutoSuggestion
+from src.api.endpoints.annotate.all.get.queries.agency.requester import GetAgencySuggestionsRequester
+from src.db.queries.base.builder import QueryBuilderBase
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.api.endpoints.annotate.all.get.models.agency import AgencyAnnotationResponseOuterInfo, \
+    AgencyAnnotationUserSuggestionOuterInfo, AgencyAnnotationUserSuggestion, AgencyAnnotationAutoSuggestion
+from src.api.endpoints.annotate.all.get.queries.agency.requester import GetAgencySuggestionsRequester
+from src.db.queries.base.builder import QueryBuilderBase
+
+
+class GetAgencySuggestionsQueryBuilder(QueryBuilderBase):
+
+    def __init__(
+        self,
+        url_id: int
+    ):
+        super().__init__()
+        self.url_id = url_id
+
+    async def run(self, session: AsyncSession) -> AgencyAnnotationResponseOuterInfo:
+        requester = GetAgencySuggestionsRequester(
+            session,
+            url_id=self.url_id
+        )
+
+        user_suggestions: list[AgencyAnnotationUserSuggestion] = \
+            await requester.get_user_agency_suggestions()
+        auto_suggestions: list[AgencyAnnotationAutoSuggestion] = \
+            await requester.get_auto_agency_suggestions()
+        not_found_count: int = \
+            await requester.get_not_found_count()
+        return AgencyAnnotationResponseOuterInfo(
+            user=AgencyAnnotationUserSuggestionOuterInfo(
+                suggestions=user_suggestions,
+                not_found_count=not_found_count
+            ),
+            auto=auto_suggestions,
+        )
+
+

src/api/endpoints/annotate/all/get/queries/agency/requester.py

@@ -0,0 +1,101 @@
+from typing import Sequence
+
+from sqlalchemy import func, select, RowMapping
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.api.endpoints.annotate.all.get.models.agency import AgencyAnnotationAutoSuggestion, \
+    AgencyAnnotationUserSuggestion
+from src.api.endpoints.annotate.all.get.queries.agency.suggestions_with_highest_confidence import \
+    SuggestionsWithHighestConfidenceCTE
+from src.db.helpers.session import session_helper as sh
+from src.db.models.impl.agency.sqlalchemy import Agency
+from src.db.models.impl.link.user_suggestion_not_found.agency.sqlalchemy import LinkUserSuggestionAgencyNotFound
+from src.db.models.impl.url.suggestion.agency.user import UserUrlAgencySuggestion
+from src.db.templates.requester import RequesterBase
+
+
+class GetAgencySuggestionsRequester(RequesterBase):
+
+    def __init__(self, session: AsyncSession, url_id: int):
+        super().__init__(session)
+        self.url_id = url_id
+
+    async def get_user_agency_suggestions(self) -> list[AgencyAnnotationUserSuggestion]:
+        query = (
+            select(
+                UserUrlAgencySuggestion.agency_id,
+                func.count(UserUrlAgencySuggestion.user_id).label("count"),
+                Agency.name.label("agency_name"),
+            )
+            .join(
+                Agency,
+                Agency.agency_id == UserUrlAgencySuggestion.agency_id
+            )
+            .where(
+                UserUrlAgencySuggestion.url_id == self.url_id
+            )
+            .group_by(
+                UserUrlAgencySuggestion.agency_id,
+                Agency.name
+            )
+            .order_by(
+                func.count(UserUrlAgencySuggestion.user_id).desc()
+            )
+            .limit(3)
+        )
+
+        results: Sequence[RowMapping] = await sh.mappings(self.session, query=query)
+
+        return [
+            AgencyAnnotationUserSuggestion(
+                agency_id=autosuggestion["agency_id"],
+                user_count=autosuggestion["count"],
+                agency_name=autosuggestion["agency_name"],
+            )
+            for autosuggestion in results
+        ]
+
+
+    async def get_auto_agency_suggestions(self) -> list[AgencyAnnotationAutoSuggestion]:
+        cte = SuggestionsWithHighestConfidenceCTE()
+        query = (
+            select(
+                cte.agency_id,
+                cte.confidence,
+                Agency.name.label("agency_name"),
+            )
+            .outerjoin(
+                Agency,
+                Agency.agency_id == cte.agency_id
+            )
+            .where(
+                cte.url_id == self.url_id
+            )
+            .order_by(
+                cte.confidence.desc()
+            )
+            .limit(3)
+        )
+
+        results: Sequence[RowMapping] = await sh.mappings(self.session, query=query)
+
+        return [
+            AgencyAnnotationAutoSuggestion(
+                agency_id=autosuggestion["agency_id"],
+                confidence=autosuggestion["confidence"],
+                agency_name=autosuggestion["agency_name"],
+            )
+            for autosuggestion in results
+        ]
+
+    async def get_not_found_count(self) -> int:
+        query = (
+            select(
+                func.count(LinkUserSuggestionAgencyNotFound.user_id)
+            )
+            .where(
+                LinkUserSuggestionAgencyNotFound.url_id == self.url_id
+            )
+        )
+
+        return await sh.scalar(self.session, query=query)