mc_125_metrics_endpoints

alembic/versions/2025_05_03_0956-028565b77b9e_add_manual_strategy_to_batch_strategy_.py

@@ -38,6 +38,12 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
+    # Delete all batches with manual strategy
+    op.execute("""
+    DELETE FROM BATCHES
+        WHERE STRATEGY = 'manual'
+    """)
+
     switch_enum_type(
         table_name="batches",
         column_name="strategy",

alembic/versions/2025_05_06_0816-e55e16e0738f_create_backlogsnapshot_table.py

@@ -0,0 +1,31 @@
+"""Create BacklogSnapshot Table
+
+Revision ID: e55e16e0738f
+Revises: 028565b77b9e
+Create Date: 2025-05-06 08:16:29.385305
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'e55e16e0738f'
+down_revision: Union[str, None] = '028565b77b9e'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        'backlog_snapshot',
+        sa.Column('id', sa.Integer(), nullable=False, primary_key=True),
+        sa.Column('count_pending_total', sa.Integer(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=False),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table('backlog_snapshot')

alembic/versions/2025_05_06_0919-f25852e17c04_create_url_annotation_flags_view.py

@@ -0,0 +1,47 @@
+"""Create URL Annotation Flags View
+
+Revision ID: f25852e17c04
+Revises: e55e16e0738f
+Create Date: 2025-05-06 09:19:54.000410
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'f25852e17c04'
+down_revision: Union[str, None] = 'e55e16e0738f'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("""
+    CREATE OR REPLACE VIEW url_annotation_flags AS
+    (
+    SELECT u.id,
+           CASE WHEN arts.url_id IS NOT NULL THEN TRUE ELSE FALSE END AS has_auto_record_type_suggestion,
+           CASE WHEN ars.url_id IS NOT NULL THEN TRUE ELSE FALSE END  AS has_auto_relevant_suggestion,
+           CASE WHEN auas.url_id IS NOT NULL THEN TRUE ELSE FALSE END AS has_auto_agency_suggestion,
+           CASE WHEN urts.url_id IS NOT NULL THEN TRUE ELSE FALSE END AS has_user_record_type_suggestion,
+           CASE WHEN urs.url_id IS NOT NULL THEN TRUE ELSE FALSE END  AS has_user_relevant_suggestion,
+           CASE WHEN uuas.url_id IS NOT NULL THEN TRUE ELSE FALSE END AS has_user_agency_suggestion,
+           CASE WHEN cua.url_id IS NOT NULL THEN TRUE ELSE FALSE END  AS has_confirmed_agency,
+           CASE WHEN ruu.url_id IS NOT NULL THEN TRUE ELSE FALSE END  AS was_reviewed
+    FROM urls u
+             LEFT JOIN public.auto_record_type_suggestions arts ON u.id = arts.url_id
+             LEFT JOIN public.auto_relevant_suggestions ars ON u.id = ars.url_id
+             LEFT JOIN public.automated_url_agency_suggestions auas ON u.id = auas.url_id
+             LEFT JOIN public.user_record_type_suggestions urts ON u.id = urts.url_id
+             LEFT JOIN public.user_relevant_suggestions urs ON u.id = urs.url_id
+             LEFT JOIN public.user_url_agency_suggestions uuas ON u.id = uuas.url_id
+             LEFT JOIN public.reviewing_user_url ruu ON u.id = ruu.url_id
+             LEFT JOIN public.confirmed_url_agency cua on u.id = cua.url_id
+    )
+    """)
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW url_annotation_flags;")

alembic/versions/2025_05_06_1115-6f2007bbcce3_create_url_data_sources_table.py

@@ -0,0 +1,79 @@
+"""Create url_data_sources table
+
+Revision ID: 6f2007bbcce3
+Revises: f25852e17c04
+Create Date: 2025-05-06 11:15:24.485465
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = '6f2007bbcce3'
+down_revision: Union[str, None] = 'f25852e17c04'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Create url_data_sources_table table
+    op.create_table(
+        'url_data_sources',
+        sa.Column(
+            'id',
+            sa.Integer(),
+            primary_key=True
+        ),
+        sa.Column(
+            'url_id',
+            sa.Integer(),
+            sa.ForeignKey(
+                'urls.id',
+                ondelete='CASCADE'
+            ),
+            nullable=False
+        ),
+        sa.Column(
+            'data_source_id',
+            sa.Integer(),
+            nullable=False
+        ),
+        sa.Column(
+            'created_at',
+            sa.TIMESTAMP(),
+            nullable=False,
+            server_default=sa.text('now()')
+        ),
+        sa.UniqueConstraint('url_id', name='uq_url_data_sources_url_id'),
+        sa.UniqueConstraint('data_source_id', name='uq_url_data_sources_data_source_id')
+    )
+
+    # Migrate existing urls with a data source ID
+    op.execute("""
+        INSERT INTO url_data_sources 
+        (url_id, data_source_id) 
+        SELECT id, data_source_id 
+        FROM urls 
+        WHERE data_source_id IS NOT NULL
+    """)
+
+    # Drop existing data source ID column from urls table
+    op.drop_column('urls', 'data_source_id')
+
+
+def downgrade() -> None:
+
+    op.drop_table('url_data_sources')
+
+    op.add_column(
+        'urls',
+        sa.Column(
+            'data_source_id',
+            sa.Integer(),
+            nullable=True
+        )
+    )
+
+

api/main.py

@@ -9,6 +9,7 @@
 from api.routes.annotate import annotate_router
 from api.routes.batch import batch_router
 from api.routes.collector import collector_router
+from api.routes.metrics import metrics_router
 from api.routes.review import review_router
 from api.routes.root import root_router
 from api.routes.search import search_router
@@ -130,7 +131,8 @@ async def redirect_docs():
     url_router,
     task_router,
     review_router,
-    search_router
+    search_router,
+    metrics_router
 ]
 
 for router in routers:

api/routes/metrics.py

@@ -0,0 +1,64 @@
+from fastapi import APIRouter
+from fastapi.params import Query, Depends
+
+from api.dependencies import get_async_core
+from core.AsyncCore import AsyncCore
+from core.DTOs.GetMetricsBacklogResponse import GetMetricsBacklogResponseDTO
+from core.DTOs.GetMetricsBatchesAggregatedResponseDTO import GetMetricsBatchesAggregatedResponseDTO
+from core.DTOs.GetMetricsBatchesBreakdownResponseDTO import GetMetricsBatchesBreakdownResponseDTO
+from core.DTOs.GetMetricsURLsAggregatedResponseDTO import GetMetricsURLsAggregatedResponseDTO
+from core.DTOs.GetMetricsURLsBreakdownPendingResponseDTO import GetMetricsURLsBreakdownPendingResponseDTO
+from core.DTOs.GetMetricsURLsBreakdownSubmittedResponseDTO import GetMetricsURLsBreakdownSubmittedResponseDTO
+from security_manager.SecurityManager import AccessInfo, get_access_info
+
+metrics_router = APIRouter(
+    prefix="/metrics",
+    tags=["Metrics"],
+)
+
+
+@metrics_router.get("/batches/aggregated")
+async def get_batches_aggregated_metrics(
+        core: AsyncCore = Depends(get_async_core),
+        access_info: AccessInfo = Depends(get_access_info)
+) -> GetMetricsBatchesAggregatedResponseDTO:
+    return await core.get_batches_aggregated_metrics()
+
+@metrics_router.get("/batches/breakdown")
+async def get_batches_breakdown_metrics(
+        core: AsyncCore = Depends(get_async_core),
+        access_info: AccessInfo = Depends(get_access_info),
+        page: int = Query(
+            description="The page number",
+            default=1
+        )
+) -> GetMetricsBatchesBreakdownResponseDTO:
+    return await core.get_batches_breakdown_metrics(page=page)
+
+@metrics_router.get("/urls/aggregate")
+async def get_urls_aggregated_metrics(
+        core: AsyncCore = Depends(get_async_core),
+        access_info: AccessInfo = Depends(get_access_info)
+) -> GetMetricsURLsAggregatedResponseDTO:
+    return await core.get_urls_aggregated_metrics()
+
+@metrics_router.get("/urls/breakdown/submitted")
+async def get_urls_breakdown_submitted_metrics(
+        core: AsyncCore = Depends(get_async_core),
+        access_info: AccessInfo = Depends(get_access_info)
+) -> GetMetricsURLsBreakdownSubmittedResponseDTO:
+    return await core.get_urls_breakdown_submitted_metrics()
+
+@metrics_router.get("/urls/breakdown/pending")
+async def get_urls_breakdown_pending_metrics(
+        core: AsyncCore = Depends(get_async_core),
+        access_info: AccessInfo = Depends(get_access_info)
+) -> GetMetricsURLsBreakdownPendingResponseDTO:
+    return await core.get_urls_breakdown_pending_metrics()
+
+@metrics_router.get("/backlog")
+async def get_backlog_metrics(
+        core: AsyncCore = Depends(get_async_core),
+        access_info: AccessInfo = Depends(get_access_info)
+) -> GetMetricsBacklogResponseDTO:
+    return await core.get_backlog_metrics()