feat(app): remove huggingface logic

Author: maxachis

Dockerfile

@@ -22,9 +22,6 @@ COPY collector_db ./collector_db
 COPY collector_manager ./collector_manager
 COPY core ./core
 COPY html_tag_collector ./html_tag_collector
-COPY hugging_face/url_relevance ./hugging_face/url_relevance
-COPY hugging_face/url_record_type_labeling ./hugging_face/url_record_type_labeling
-COPY hugging_face/HuggingFaceInterface.py ./hugging_face/HuggingFaceInterface.py
 COPY source_collectors ./source_collectors
 COPY util ./util
 COPY alembic.ini ./alembic.ini

README.md

@@ -8,7 +8,6 @@ name | description of purpose
 agency_identifier | Matches URLs with an agency from the PDAP database
 annotation_pipeline | Automated pipeline for generating training data in our ML data source identification models. Manages common crawl, HTML tag collection, and Label Studio import/export
 html_tag_collector | Collects HTML header, meta, and title tags and appends them to a JSON file. The idea is to make a richer dataset for algorithm training and data labeling.
-hugging_face | Utilities for interacting with our machine learning space at [Hugging Face](https://huggingface.co/PDAP)
 identification_pipeline.py | The core python script uniting this modular pipeline. More details below.
 openai-playground | Scripts for accessing the openai API on PDAP's shared account
 source_collectors| Tools for extracting metadata from different sources, including CKAN data portals and Common Crawler

api/main.py

@@ -26,7 +26,6 @@
 from html_tag_collector.ResponseParser import HTMLResponseParser
 from html_tag_collector.RootURLCache import RootURLCache
 from html_tag_collector.URLRequestInterface import URLRequestInterface
-from hugging_face.HuggingFaceInterface import HuggingFaceInterface
 from pdap_access_manager import AccessManager
 from pdap_api_client.PDAPClient import PDAPClient
 from util.DiscordNotifier import DiscordPoster
@@ -54,7 +53,6 @@ async def lifespan(app: FastAPI):
     )
     task_manager = TaskManager(
         adb_client=adb_client,
-        huggingface_interface=HuggingFaceInterface(),
         url_request_interface=URLRequestInterface(),
         html_parser=HTMLResponseParser(
             root_url_cache=RootURLCache()

core/DTOs/task_data_objects/URLRelevanceHuggingfaceTDO.py

@@ -15,11 +15,9 @@
 from core.classes.task_operators.URLHTMLTaskOperator import URLHTMLTaskOperator
 from core.classes.task_operators.URLMiscellaneousMetadataTaskOperator import URLMiscellaneousMetadataTaskOperator
 from core.classes.task_operators.URLRecordTypeTaskOperator import URLRecordTypeTaskOperator
-from core.classes.task_operators.URLRelevanceHuggingfaceTaskOperator import URLRelevanceHuggingfaceTaskOperator
 from core.enums import BatchStatus
 from html_tag_collector.ResponseParser import HTMLResponseParser
 from html_tag_collector.URLRequestInterface import URLRequestInterface
-from hugging_face.HuggingFaceInterface import HuggingFaceInterface
 from llm_api_logic.OpenAIRecordClassifier import OpenAIRecordClassifier
 from pdap_api_client.PDAPClient import PDAPClient
 from util.DiscordNotifier import DiscordPoster
@@ -31,16 +29,14 @@ class TaskManager:
     def __init__(
             self,
             adb_client: AsyncDatabaseClient,
-            huggingface_interface: HuggingFaceInterface,
             url_request_interface: URLRequestInterface,
             html_parser: HTMLResponseParser,
             discord_poster: DiscordPoster,
-            pdap_client: PDAPClient
+            pdap_client: PDAPClient,
     ):
         # Dependencies
         self.adb_client = adb_client
         self.pdap_client = pdap_client
-        self.huggingface_interface = huggingface_interface
         self.url_request_interface = url_request_interface
         self.html_parser = html_parser
         self.discord_poster = discord_poster
@@ -62,13 +58,6 @@ async def get_url_html_task_operator(self):
         )
         return operator
 
-    async def get_url_relevance_huggingface_task_operator(self):
-        operator = URLRelevanceHuggingfaceTaskOperator(
-            adb_client=self.adb_client,
-            huggingface_interface=self.huggingface_interface
-        )
-        return operator
-
     async def get_url_record_type_task_operator(self):
         operator = URLRecordTypeTaskOperator(
             adb_client=self.adb_client,
@@ -117,7 +106,6 @@ async def get_task_operators(self) -> list[TaskOperatorBase]:
             await self.get_url_html_task_operator(),
             await self.get_url_duplicate_task_operator(),
             await self.get_url_404_probe_task_operator(),
-            # await self.get_url_relevance_huggingface_task_operator(),
             await self.get_url_record_type_task_operator(),
             await self.get_agency_identification_task_operator(),
             await self.get_url_miscellaneous_metadata_task_operator(),