Merge pull request #438 from Police-Data-Accessibility-Project/mc_fix_annotation_bug

mc_fix_annotation_bug

Author: maxachis

ENV.md

@@ -85,6 +85,7 @@ URL Task Flags are collectively controlled by the `RUN_URL_TASKS_TASK_FLAG` flag
 | `URL_ROOT_URL_TASK_FLAG`            | Extracts and links Root URLs to URLs.                  |
 | `URL_SCREENSHOT_TASK_FLAG`          | Takes screenshots of URLs.                             |
 | `URL_AUTO_VALIDATE_TASK_FLAG`       | Automatically validates URLs.                          |
+| `URL_AUTO_NAME_TASK_FLAG`           | Automatically names URLs.                              |
 
 ### Agency ID Subtasks
 

alembic/versions/2025_09_24_1739-3687026267fc_add_url_naming_logic.py

@@ -0,0 +1,69 @@
+"""Add URL naming logic
+
+Revision ID: 3687026267fc
+Revises: e6a1a1b3bad4
+Create Date: 2025-09-24 17:39:55.353947
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from src.util.alembic_helpers import id_column, url_id_column, created_at_column, user_id_column
+
+# revision identifiers, used by Alembic.
+revision: str = '3687026267fc'
+down_revision: Union[str, None] = 'e6a1a1b3bad4'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+
+def upgrade() -> None:
+    _add_auto_name_task()
+    _create_url_name_suggestion_table()
+    _create_link_user_name_suggestion_table()
+
+def _add_auto_name_task():
+    op.execute("""ALTER TYPE task_type ADD VALUE 'Auto Name';""")
+
+
+def _create_url_name_suggestion_table():
+    op.create_table(
+        'url_name_suggestions',
+        id_column(),
+        url_id_column(),
+        sa.Column('suggestion', sa.String(
+            length=100
+        ), nullable=False),
+        sa.Column(
+            'source', sa.Enum(
+                "HTML Metadata Title",
+                "User",
+                name="suggestion_source_enum"
+            )
+        ),
+        created_at_column(),
+        sa.UniqueConstraint(
+            'url_id', 'suggestion', name='url_name_suggestions_url_id_source_unique'
+        )
+    )
+
+
+def _create_link_user_name_suggestion_table():
+    op.create_table(
+        'link_user_name_suggestions',
+        user_id_column(),
+        sa.Column(
+            "suggestion_id",
+            sa.Integer(),
+            sa.ForeignKey("url_name_suggestions.id"),
+            nullable=False,
+        ),
+        created_at_column(),
+        sa.PrimaryKeyConstraint(
+            "user_id",
+            "suggestion_id"
+        )
+    )

src/core/tasks/url/loader.py

@@ -8,6 +8,7 @@
 from src.core.tasks.url.models.entry import URLTaskEntry
 from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator
 from src.core.tasks.url.operators.agency_identification.subtasks.loader import AgencyIdentificationSubtaskLoader
+from src.core.tasks.url.operators.auto_name.core import AutoNameURLTaskOperator
 from src.core.tasks.url.operators.auto_relevant.core import URLAutoRelevantTaskOperator
 from src.core.tasks.url.operators.html.core import URLHTMLTaskOperator
 from src.core.tasks.url.operators.html.scraper.parser.core import HTMLResponseParser
@@ -213,6 +214,18 @@ def _get_auto_validate_task_operator(self) -> URLTaskEntry:
             )
         )
 
+    def _get_auto_name_task_operator(self) -> URLTaskEntry:
+        operator = AutoNameURLTaskOperator(
+            adb_client=self.adb_client,
+        )
+        return URLTaskEntry(
+            operator=operator,
+            enabled=self.env.bool(
+                "URL_AUTO_NAME_TASK_FLAG",
+                default=True
+            )
+        )
+
 
     async def load_entries(self) -> list[URLTaskEntry]:
         return [
@@ -227,5 +240,6 @@ async def load_entries(self) -> list[URLTaskEntry]:
             self._get_url_auto_relevance_task_operator(),
             self._get_url_screenshot_task_operator(),
             self._get_location_id_task_operator(),
-            self._get_auto_validate_task_operator()
+            self._get_auto_validate_task_operator(),
+            self._get_auto_name_task_operator(),
         ]

src/core/tasks/url/operators/auto_name/__init__.py

@@ -0,0 +1,7 @@
+from src.db.models.impl.url.suggestion.location.auto.subtask.constants import MAX_SUGGESTION_LENGTH
+
+
+def clean_title(title: str) -> str:
+    if len(title) > MAX_SUGGESTION_LENGTH:
+        return title[:MAX_SUGGESTION_LENGTH-3] + "..."
+    return title

src/core/tasks/url/operators/auto_name/clean.py

@@ -0,0 +1,44 @@
+from src.core.tasks.url.operators.auto_name.clean import clean_title
+from src.core.tasks.url.operators.auto_name.input import AutoNamePrerequisitesInput
+from src.core.tasks.url.operators.auto_name.queries.get import AutoNameGetInputsQueryBuilder
+from src.core.tasks.url.operators.auto_name.queries.prereq import AutoNamePrerequisitesQueryBuilder
+from src.core.tasks.url.operators.base import URLTaskOperatorBase
+from src.db.enums import TaskType
+from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource
+from src.db.models.impl.url.suggestion.name.pydantic import URLNameSuggestionPydantic
+
+
+class AutoNameURLTaskOperator(URLTaskOperatorBase):
+
+    @property
+    def task_type(self) -> TaskType:
+        return TaskType.AUTO_NAME
+
+    async def meets_task_prerequisites(self) -> bool:
+        return await self.adb_client.run_query_builder(
+            AutoNamePrerequisitesQueryBuilder()
+        )
+
+    async def inner_task_logic(self) -> None:
+
+        # Get URLs with HTML metadata title
+        inputs: list[AutoNamePrerequisitesInput] = await self.adb_client.run_query_builder(
+            AutoNameGetInputsQueryBuilder()
+        )
+
+        # Link URLs to task
+        url_ids: list[int] = [input.url_id for input in inputs]
+        await self.link_urls_to_task(url_ids)
+
+        # Add suggestions
+        suggestions: list[URLNameSuggestionPydantic] = [
+            URLNameSuggestionPydantic(
+                url_id=input_.url_id,
+                suggestion=clean_title(input_.title),
+                source=NameSuggestionSource.HTML_METADATA_TITLE,
+            )
+            for input_ in inputs
+        ]
+
+        await self.adb_client.bulk_insert(models=suggestions)
+

src/core/tasks/url/operators/auto_name/core.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class AutoNamePrerequisitesInput(BaseModel):
+    url_id: int
+    title: str

src/core/tasks/url/operators/auto_name/input.py

@@ -0,0 +1,46 @@
+from sqlalchemy import select, exists, CTE, Column
+
+from src.db.enums import URLHTMLContentType
+from src.db.models.impl.url.core.sqlalchemy import URL
+from src.db.models.impl.url.html.content.sqlalchemy import URLHTMLContent
+from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource
+from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion
+
+
+class AutoNamePrerequisiteCTEContainer:
+
+    def __init__(self):
+        self._query = (
+            select(
+                URL.id.label("url_id"),
+                URLHTMLContent.content
+            )
+            .join(
+                URLHTMLContent,
+                URLHTMLContent.url_id == URL.id
+            )
+            .where(
+                URLHTMLContent.content_type == URLHTMLContentType.TITLE.value,
+                ~exists(
+                    select(
+                        URLNameSuggestion.id
+                    )
+                    .where(
+                        URLNameSuggestion.url_id == URL.id,
+                        URLNameSuggestion.source == NameSuggestionSource.HTML_METADATA_TITLE.value,
+                    )
+                )
+            ).cte("auto_name_prerequisites")
+        )
+
+    @property
+    def cte(self) -> CTE:
+        return self._query
+
+    @property
+    def url_id(self) -> Column[int]:
+        return self.cte.c.url_id
+
+    @property
+    def content(self) -> Column[str]:
+        return self.cte.c.content

src/core/tasks/url/operators/auto_name/queries/__init__.py

@@ -0,0 +1,27 @@
+from typing import Sequence
+
+from sqlalchemy import select, RowMapping
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.core.tasks.url.operators.auto_name.input import AutoNamePrerequisitesInput
+from src.core.tasks.url.operators.auto_name.queries.cte import AutoNamePrerequisiteCTEContainer
+from src.db.queries.base.builder import QueryBuilderBase
+
+from src.db.helpers.session import session_helper as sh
+
+class AutoNameGetInputsQueryBuilder(QueryBuilderBase):
+
+    async def run(self, session: AsyncSession) -> list[AutoNamePrerequisitesInput]:
+        cte = AutoNamePrerequisiteCTEContainer()
+        query = select(cte.url_id, cte.content)
+
+        mappings: Sequence[RowMapping] = await sh.mappings(session=session, query=query)
+        results: list[AutoNamePrerequisitesInput] = []
+        for mapping in mappings:
+            result = AutoNamePrerequisitesInput(
+                url_id=mapping["url_id"],
+                title=mapping["content"],
+            )
+            results.append(result)
+
+        return results