mc_381_url_agency_identifier

Dockerfile

@@ -14,6 +14,8 @@ RUN uv sync --locked --no-dev
 # Must call from the root directory because uv does not add playwright to path
 RUN playwright install-deps chromium
 RUN playwright install chromium
+# Download Spacy Model
+RUN python -m spacy download en_core_web_sm
 
 # Copy project files
 COPY src ./src

ENV.md

@@ -53,7 +53,10 @@ The following flags are available:
 | `RUN_URL_TASKS_TASK_FLAG`           | Runs URL tasks.                                        |
 | `IA_PROBE_TASK_FLAG`            | Extracts and links Internet Archives metadata to URLs. |
 | `IA_SAVE_TASK_FLAG`             | Saves URLs to Internet Archives.                       |
-
+| `AGENCY_ID_HOMEPAGE_MATCH_FLAG` | Enables the homepage match subtask for agency identification. |
+| `AGENCY_ID_NLP_LOCATION_MATCH_FLAG` | Enables the NLP location match subtask for agency identification. |
+| `AGENCY_ID_CKAN_FLAG` | Enables the CKAN subtask for agency identification. |
+| `AGENCY_ID_MUCKROCK_FLAG` | Enables the MuckRock subtask for agency identification. |
 
 
 ## Foreign Data Wrapper (FDW)

alembic/versions/2025_08_09_2031-8cd5aa7670ff_remove_functional_duplicates.py

@@ -8,7 +8,7 @@
 from typing import Sequence, Union
 
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
@@ -33,7 +33,7 @@
 AUTOMATED_URL_AGENCY_SUGGESTION_FOREIGN_KEY_NAME = 'automated_url_agency_suggestions_url_id_fkey'
 
 
 def upgrade() -> None:
     _add_cascade_foreign_key(URL_HTML_CONTENT_TABLE_NAME, foreign_key_name=URL_HTML_CONTENT_FOREIGN_KEY_NAME)
     _add_cascade_foreign_key(COMPRESSED_HTML_TABLE_NAME, foreign_key_name=COMPRESSED_HTML_FOREIGN_KEY_NAME)
     _add_cascade_foreign_key(URL_ERROR_INFO_TABLE_NAME, foreign_key_name=URL_ERROR_INFO_FOREIGN_KEY_NAME)
@@ -46,13 +46,13 @@
     _add_constraint_forbidding_fragments()
 
 
 def downgrade() -> None:
     _remove_constraint_forbidding_fragments()
     _remove_constraint_forbidding_nbsp()
     _remove_cascade_foreign_key(URL_ERROR_INFO_TABLE_NAME, foreign_key_name=URL_ERROR_INFO_FOREIGN_KEY_NAME)
     _remove_cascade_foreign_key(COMPRESSED_HTML_TABLE_NAME, foreign_key_name=COMPRESSED_HTML_FOREIGN_KEY_NAME)
     _remove_cascade_foreign_key(URL_HTML_CONTENT_TABLE_NAME, foreign_key_name=URL_HTML_CONTENT_FOREIGN_KEY_NAME)
-    _remove_cascade_foreign_key(AUTOMATED_URL_AGENCY_SUGGESTION_TABLE_NAME, foreign_key_name=AUTOMATED_URL_AGENCY_SUGGESTION_FOREIGN_KEY_NAME)
+    # _remove_cascade_foreign_key(AUTOMATED_URL_AGENCY_SUGGESTION_TABLE_NAME, foreign_key_name=AUTOMATED_URL_AGENCY_SUGGESTION_FOREIGN_KEY_NAME)
 
 def _delete_duplicate_urls() -> None:
     op.execute('delete from urls where id in (2341,2343,2344,2347,2348,2349,2354,2359,2361,2501,2504,2505,2506,2507)')
@@ -87,13 +87,13 @@
     delete from urls
     where source = 'data_sources_app'
     """
     )
 
 def _reset_data_sources_sync_state() -> None:
     op.execute("""
     delete from data_sources_sync_state
     """
    )
 
 def _add_constraint_forbidding_nbsp() -> None:
     op.create_check_constraint(
@@ -121,3 +121,3 @@
     update urls
     set url = substring(url from 1 for position('#' in url) - 1)
     where url like '%#%'

alembic/versions/2025_08_19_0803-b741b65a1431_augment_auto_agency_suggestions.py

@@ -0,0 +1,254 @@
+"""Augment auto_agency_suggestions
+
+Revision ID: b741b65a1431
+Revises: 8a70ee509a74
+Create Date: 2025-08-19 08:03:12.106575
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from src.util.alembic_helpers import created_at_column, updated_at_column, id_column, url_id_column, switch_enum_type
+
+# revision identifiers, used by Alembic.
+revision: str = 'b741b65a1431'
+down_revision: Union[str, None] = '8a70ee509a74'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+OLD_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME = "automated_url_agency_suggestions"
+NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME = "url_auto_agency_suggestions"
+
+OLD_LINK_URLS_AGENCY_TABLE_NAME = "link_urls_agencies"
+NEW_LINK_URLS_AGENCY_TABLE_NAME = "link_urls_agency"
+
+AGENCY_AUTO_SUGGESTION_METHOD_ENUM = sa.Enum(
+    "homepage_match",
+    "nlp_location_match",
+    "muckrock_match",
+    "ckan_match",
+    name="agency_auto_suggestion_method",
+)
+
+FLAG_URL_VALIDATED_TABLE_NAME = "flag_url_validated"
+
+VALIDATED_URL_TYPE_ENUM = sa.Enum(
+    "data source",
+    "meta url",
+    "not relevant",
+    "individual record",
+    name="validated_url_type"
+)
+
+
+
+
+def upgrade() -> None:
+    op.rename_table(OLD_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME, NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME)
+    op.rename_table(OLD_LINK_URLS_AGENCY_TABLE_NAME, NEW_LINK_URLS_AGENCY_TABLE_NAME)
+    _alter_auto_agency_suggestions_table()
+    _create_flag_url_validated_table()
+    _add_urls_to_flag_url_validated_table()
+    _remove_validated_and_submitted_url_statuses()
+    _reset_agencies_sync_state()
+
+
+def downgrade() -> None:
+    op.rename_table(NEW_LINK_URLS_AGENCY_TABLE_NAME, OLD_LINK_URLS_AGENCY_TABLE_NAME)
+    _revert_auto_agency_suggestions_table()
+    op.rename_table(NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME, OLD_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME)
+    _revert_url_statuses()
+    _update_validated_and_submitted_url_statuses()
+    op.drop_table(FLAG_URL_VALIDATED_TABLE_NAME)
+    _drop_validated_url_type_enum()
+
+def _reset_agencies_sync_state():
+    op.execute(
+        """
+        UPDATE agencies_sync_state
+        set
+            last_full_sync_at = null,
+            current_cutoff_date = null,
+            current_page = null
+        """
+    )
+
+def _remove_validated_and_submitted_url_statuses():
+    switch_enum_type(
+        table_name="urls",
+        column_name="status",
+        enum_name="url_status",
+        new_enum_values=[
+            'ok',
+            'duplicate',
+            'error',
+            '404 not found',
+        ],
+        check_constraints_to_drop=['url_name_not_null_when_validated'],
+        conversion_mappings={
+            'validated': 'ok',
+            'submitted': 'ok',
+            'pending': 'ok',
+            'not relevant': 'ok',
+            'individual record': 'ok'
+        }
+    )
+
+def _add_urls_to_flag_url_validated_table():
+    op.execute("""
+    INSERT INTO flag_url_validated (url_id, type)
+    SELECT 
+        urls.id, 
+           CASE urls.status::text
+               WHEN 'validated' THEN 'data source'
+               WHEN 'submitted' THEN 'data source'
+               ELSE urls.status::text
+           END::validated_url_type
+    FROM urls
+    WHERE urls.status in ('validated', 'submitted', 'individual record', 'not relevant')""")
+
+def _revert_url_statuses():
+    switch_enum_type(
+        table_name="urls",
+        column_name="status",
+        enum_name="url_status",
+        new_enum_values=[
+            'pending',
+            'validated',
+            'submitted',
+            'duplicate',
+            'not relevant',
+            'error',
+            '404 not found',
+            'individual record'
+        ],
+        conversion_mappings={
+            'ok': 'pending',
+        }
+    )
+    op.create_check_constraint(
+        "url_name_not_null_when_validated",
+        "urls",
+        "(name IS NOT NULL) OR (status <> 'validated'::url_status)"
+    )
+
+def _update_validated_and_submitted_url_statuses():
+    op.execute("""
+    UPDATE urls
+    SET status = 'not relevant'
+    FROM flag_url_validated
+    WHERE urls.id = flag_url_validated.id
+    AND flag_url_validated.type = 'not relevant'
+    """)
+
+    op.execute("""
+    UPDATE urls
+    SET status = 'individual record'
+    FROM flag_url_validated
+    WHERE urls.id = flag_url_validated.id
+    AND flag_url_validated.type = 'individual record'
+    """)
+
+    op.execute("""
+    UPDATE urls
+    SET status = 'validated'
+    FROM flag_url_validated
+    left join url_data_source on flag_url_validated.url_id = url_data_source.url_id
+    WHERE urls.id = flag_url_validated.id
+    AND flag_url_validated.type = 'data source'
+    AND url_data_source.url_id is NULL
+    """)
+
+    op.execute("""
+    UPDATE urls
+    SET status = 'validated'
+    FROM flag_url_validated
+    left join url_data_source on flag_url_validated.url_id = url_data_source.url_id
+    WHERE urls.id = flag_url_validated.id
+    AND flag_url_validated.type = 'data source'
+    AND url_data_source.url_id is not NULL
+    """)
+
+
+def _create_flag_url_validated_table():
+    op.create_table(
+        FLAG_URL_VALIDATED_TABLE_NAME,
+        id_column(),
+        url_id_column(),
+        sa.Column(
+            'type',
+            VALIDATED_URL_TYPE_ENUM,
+            nullable=False,
+        ),
+        created_at_column(),
+        updated_at_column(),
+        sa.UniqueConstraint('url_id', name='uq_flag_url_validated_url_id')
+    )
+
+def _drop_validated_url_type_enum():
+    VALIDATED_URL_TYPE_ENUM.drop(op.get_bind())
+
+def _alter_auto_agency_suggestions_table():
+    AGENCY_AUTO_SUGGESTION_METHOD_ENUM.create(op.get_bind())
+    # Created At
+    op.add_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        created_at_column()
+    )
+    # Updated At
+    op.add_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        updated_at_column()
+    )
+    # Method
+    op.add_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        sa.Column(
+            'method',
+            AGENCY_AUTO_SUGGESTION_METHOD_ENUM,
+            nullable=True
+        )
+    )
+    # Confidence
+    op.add_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        sa.Column(
+            'confidence',
+            sa.Float(),
+            server_default=sa.text('0.0'),
+            nullable=False
+        )
+    )
+    # Check constraint that confidence is between 0 and 1
+    op.create_check_constraint(
+        "auto_url_agency_suggestions_check_confidence_between_0_and_1",
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        "confidence BETWEEN 0 AND 1"
+    )
+
+
+def _revert_auto_agency_suggestions_table():
+    # Created At
+    op.drop_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        'created_at'
+    )
+    # Updated At
+    op.drop_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        'updated_at'
+    )
+    # Method
+    op.drop_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        'method'
+    )
+    # Confidence
+    op.drop_column(
+        NEW_AUTO_URL_AGENCY_SUGGESTIONS_TABLE_NAME,
+        'confidence'
+    )
+    AGENCY_AUTO_SUGGESTION_METHOD_ENUM.drop(op.get_bind())
+