mc_255_remove_huggingface_logic

Dockerfile

@@ -22,9 +22,6 @@ COPY collector_db ./collector_db
 COPY collector_manager ./collector_manager
 COPY core ./core
 COPY html_tag_collector ./html_tag_collector
-COPY hugging_face/url_relevance ./hugging_face/url_relevance
-COPY hugging_face/url_record_type_labeling ./hugging_face/url_record_type_labeling
-COPY hugging_face/HuggingFaceInterface.py ./hugging_face/HuggingFaceInterface.py
 COPY source_collectors ./source_collectors
 COPY util ./util
 COPY alembic.ini ./alembic.ini

README.md

@@ -8,7 +8,6 @@ name | description of purpose
 agency_identifier | Matches URLs with an agency from the PDAP database
 annotation_pipeline | Automated pipeline for generating training data in our ML data source identification models. Manages common crawl, HTML tag collection, and Label Studio import/export
 html_tag_collector | Collects HTML header, meta, and title tags and appends them to a JSON file. The idea is to make a richer dataset for algorithm training and data labeling.
-hugging_face | Utilities for interacting with our machine learning space at [Hugging Face](https://huggingface.co/PDAP)
 identification_pipeline.py | The core python script uniting this modular pipeline. More details below.
 openai-playground | Scripts for accessing the openai API on PDAP's shared account
 source_collectors| Tools for extracting metadata from different sources, including CKAN data portals and Common Crawler

api/main.py

@@ -26,7 +26,6 @@
 from html_tag_collector.ResponseParser import HTMLResponseParser
 from html_tag_collector.RootURLCache import RootURLCache
 from html_tag_collector.URLRequestInterface import URLRequestInterface
-from hugging_face.HuggingFaceInterface import HuggingFaceInterface
 from pdap_access_manager import AccessManager
 from pdap_api_client.PDAPClient import PDAPClient
 from util.DiscordNotifier import DiscordPoster
@@ -54,7 +53,6 @@
     )
     task_manager = TaskManager(
         adb_client=adb_client,
-        huggingface_interface=HuggingFaceInterface(),
         url_request_interface=URLRequestInterface(),
         html_parser=HTMLResponseParser(
             root_url_cache=RootURLCache()
@@ -118,7 +116,7 @@
 )
 
 @app.get("/docs", include_in_schema=False)
 async def redirect_docs():
     return RedirectResponse(url="/api")
 
 
@@ -139,3 +137,3 @@
 
 
 if __name__ == "__main__":

core/TaskManager.py

@@ -1,4 +1,4 @@
 import logging
 
 from core.classes.task_operators.URL404ProbeTaskOperator import URL404ProbeTaskOperator
 from core.classes.task_operators.URLDuplicateTaskOperator import URLDuplicateTaskOperator
@@ -15,32 +15,28 @@
 from core.classes.task_operators.URLHTMLTaskOperator import URLHTMLTaskOperator
 from core.classes.task_operators.URLMiscellaneousMetadataTaskOperator import URLMiscellaneousMetadataTaskOperator
 from core.classes.task_operators.URLRecordTypeTaskOperator import URLRecordTypeTaskOperator
-from core.classes.task_operators.URLRelevanceHuggingfaceTaskOperator import URLRelevanceHuggingfaceTaskOperator
 from core.enums import BatchStatus
 from html_tag_collector.ResponseParser import HTMLResponseParser
 from html_tag_collector.URLRequestInterface import URLRequestInterface
-from hugging_face.HuggingFaceInterface import HuggingFaceInterface
 from llm_api_logic.OpenAIRecordClassifier import OpenAIRecordClassifier
 from pdap_api_client.PDAPClient import PDAPClient
 from util.DiscordNotifier import DiscordPoster
 
 TASK_REPEAT_THRESHOLD = 20
 
 class TaskManager:
 
     def __init__(
             self,
             adb_client: AsyncDatabaseClient,
-            huggingface_interface: HuggingFaceInterface,
             url_request_interface: URLRequestInterface,
             html_parser: HTMLResponseParser,
             discord_poster: DiscordPoster,
-            pdap_client: PDAPClient
+            pdap_client: PDAPClient,
     ):
         # Dependencies
         self.adb_client = adb_client
         self.pdap_client = pdap_client
-        self.huggingface_interface = huggingface_interface
         self.url_request_interface = url_request_interface
         self.html_parser = html_parser
         self.discord_poster = discord_poster
@@ -53,8 +49,8 @@
 
 
 
     #region Task Operators
     async def get_url_html_task_operator(self):
         operator = URLHTMLTaskOperator(
             adb_client=self.adb_client,
             url_request_interface=self.url_request_interface,
@@ -62,21 +58,14 @@
         )
         return operator
 
-    async def get_url_relevance_huggingface_task_operator(self):
-        operator = URLRelevanceHuggingfaceTaskOperator(
-            adb_client=self.adb_client,
-            huggingface_interface=self.huggingface_interface
-        )
-        return operator
-
     async def get_url_record_type_task_operator(self):
         operator = URLRecordTypeTaskOperator(
             adb_client=self.adb_client,
             classifier=OpenAIRecordClassifier()
         )
         return operator
 
     async def get_agency_identification_task_operator(self):
         muckrock_api_interface = MuckrockAPIInterface()
         operator = AgencyIdentificationTaskOperator(
             adb_client=self.adb_client,
@@ -85,52 +74,51 @@
         )
         return operator
 
     async def get_submit_approved_url_task_operator(self):
         operator = SubmitApprovedURLTaskOperator(
             adb_client=self.adb_client,
             pdap_client=self.pdap_client
         )
         return operator
 
     async def get_url_miscellaneous_metadata_task_operator(self):
         operator = URLMiscellaneousMetadataTaskOperator(
             adb_client=self.adb_client
         )
         return operator
 
     async def get_url_duplicate_task_operator(self):
         operator = URLDuplicateTaskOperator(
             adb_client=self.adb_client,
             pdap_client=self.pdap_client
         )
         return operator
 
     async def get_url_404_probe_task_operator(self):
         operator = URL404ProbeTaskOperator(
             adb_client=self.adb_client,
             url_request_interface=self.url_request_interface
         )
         return operator
 
     async def get_task_operators(self) -> list[TaskOperatorBase]:
         return [
             await self.get_url_html_task_operator(),
             await self.get_url_duplicate_task_operator(),
             await self.get_url_404_probe_task_operator(),
-            # await self.get_url_relevance_huggingface_task_operator(),
             await self.get_url_record_type_task_operator(),
             await self.get_agency_identification_task_operator(),
             await self.get_url_miscellaneous_metadata_task_operator(),
             await self.get_submit_approved_url_task_operator()
         ]
 
     #endregion
 
     #region Tasks
     async def set_task_status(self, task_type: TaskType):
         self.task_status = task_type
 
     async def run_tasks(self):
         operators = await self.get_task_operators()
         for operator in operators:
             count = 0
@@ -153,23 +141,23 @@
                 meets_prereq = await operator.meets_task_prerequisites()
         await self.set_task_status(task_type=TaskType.IDLE)
 
     async def trigger_task_run(self):
         await self.task_trigger.trigger_or_rerun()
 
 
     async def conclude_task(self, run_info):
         await self.adb_client.link_urls_to_task(
             task_id=run_info.task_id,
             url_ids=run_info.linked_url_ids
         )
         await self.handle_outcome(run_info)
 
     async def initiate_task_in_db(self, task_type: TaskType) -> int:
         self.logger.info(f"Initiating {task_type.value} Task")
         task_id = await self.adb_client.initiate_task(task_type=task_type)
         return task_id
 
     async def handle_outcome(self, run_info: TaskOperatorRunInfo):
         match run_info.outcome:
             case TaskOperatorOutcome.ERROR:
                 await self.handle_task_error(run_info)
@@ -179,7 +167,7 @@
                     status=BatchStatus.READY_TO_LABEL
                 )
 
     async def handle_task_error(self, run_info: TaskOperatorRunInfo):
         await self.adb_client.update_task_status(
             task_id=run_info.task_id,
             status=BatchStatus.ERROR)
@@ -190,10 +178,10 @@
         await self.discord_poster.post_to_discord(
             message=f"Task {run_info.task_id} ({self.task_status.value}) failed with error.")
 
     async def get_task_info(self, task_id: int) -> TaskInfo:
         return await self.adb_client.get_task_info(task_id=task_id)
 
     async def get_tasks(
             self,
             page: int,
             task_type: TaskType,
@@ -206,7 +194,7 @@
         )
 
 
     #endregion