Mc 128 source collector api

.github/workflows/test_app.yml

@@ -0,0 +1,18 @@
+# This workflow will test the Source Collector App
+# Utilizing the docker-compose file in the root directory
+name: Test Source Collector App
+on: pull_request
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+            - name: Run docker-compose
+              uses: hoverkraft-tech/compose-action@v2.0.1
+              with:
+                  compose-file: "docker-compose.yml"
+            - name: Execute tests in the running service
+              run: |
+                  docker exec data-source-identification-app-1 pytest /app/tests/test_automated

Dockerfile

@@ -0,0 +1,18 @@
+# Dockerfile for Source Collector FastAPI app
+
+FROM python:3.12.8
+
+# Set working directory
+WORKDIR /app
+
+# Copy project files
+COPY . .
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose the application port
+EXPOSE 80
+
+# Run FastAPI app with uvicorn
+CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "80"]

ENV.md

@@ -0,0 +1,17 @@
+This page provides a full list, with description, of all the environment variables used by the application.
+
+Please ensure these are properly defined in a `.env` file in the root directory.
+
+| Name                           | Description                                                                                                                                                                                                                                                       | Example  |
+|--------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
+| `LABEL_STUDIO_ACCESS_TOKEN`    | The access token for the Label Studio API. The access token for the Label Studio API. This can be obtained by logging into Label Studio and navigating to the [user account section](https://app.heartex.com/user/account), where the access token can be copied. | `abc123` |
+| `LABEL_STUDIO_PROJECT_ID`      | The project ID for the Label Studio API. This can be obtained by logging into Label Studio and navigating to the relevant project, where the project id will be in the URL, as in `https://app.heartex.com/projects/58475/`                                       | `58475`  |
+| `LABEL_STUDIO_ORGANIZATION_ID` | The organization ID for the Label Studio API. This can be obtained by logging into Label Studio and navigating to the [Organization section](https://app.heartex.com/organization?page=1), where the organization ID can be copied.                               | `6758`   |
+| `GOOGLE_API_KEY`               | The API key required for accessing the Google Custom Search API                                                                                                                                                                                                   | `abc123` |
+| `GOOGLE_CSE_ID`                | The CSE ID required for accessing the Google Custom Search API                                                                                                                                                                                                    | `abc123` |
+|`POSTGRES_USER`                 | The username for the test database                                                                                                                                                                                                                                | `test_source_collector_user` |
+|`POSTGRES_PASSWORD`             | The password for the test database                                                                                                                                                                                                                                | `HanviliciousHamiltonHilltops` |
+|`POSTGRES_DB`                   | The database name for the test database                                                                                                                                                                                                                           | `source_collector_test_db` |
+|`POSTGRES_HOST`                 | The host for the test database                                                                                                                                                                                                                                    | `127.0.0.1` |
+|`POSTGRES_PORT`                 | The port for the test database                                                                                                                                                                                                                                    | `5432` |
+|`DS_APP_SECRET_KEY`| The secret key used for decoding JWT tokens produced by the Data Sources App. Must match the secret token that is used in the Data Sources App for encoding.                                                                                                      |`abc123`|

README.md

@@ -14,6 +14,9 @@ openai-playground | Scripts for accessing the openai API on PDAP's shared accoun
 source_collectors| Tools for extracting metadata from different sources, including CKAN data portals and Common Crawler
 collector_db | Database for storing data from source collectors
 collector_manager | A module which provides a unified interface for interacting with source collectors and relevant data
+core | A module which integrates other components, such as collector_manager and collector_db
+api | API for interacting with collector_manager, core, and collector_db
+local_database | Resources for setting up a test database for local development
 
 ## How to use
 
@@ -30,6 +33,29 @@ Thank you for your interest in contributing to this project! Please follow these
 - If you want to work on something, create an issue first so the broader community can discuss it.
 - If you make a utility, script, app, or other useful bit of code: put it in a top-level directory with an appropriate name and dedicated README and add it to the index.
 
+# Testing
+
+Note that prior to running tests, you need to install [Docker](https://docs.docker.com/get-started/get-docker/) and have the Docker engine running.
+
+Tests can be run by spinning up the `docker-compose-test.yml` file in the root directory. This will start a two-container setup, consisting of the FastAPI Web App and a clean Postgres Database. 
+
+This can be done via the following command:
+
+```bash
+docker compose up -d
+```
+
+Note that while the container may mention the web app running on `0.0.0.0:8000`, the actual host may be `127.0.0.1:8000`.
+
+To access the API documentation, visit `http://{host}:8000/docs`.
+
+To run tests on the container, run:
+
+```bash
+docker exec data-source-identification-app-1 pytest /app/tests/test_automated
+```
+
+Be sure to inspect the `docker-compose.yml` file in the root directory -- some environment variables are dependant upon the Operating System you are using.
 
 # Diagrams
 

annotation_pipeline/populate_labelstudio.py

@@ -202,7 +202,7 @@ def label_studio_upload(batch_id: str, FILENAME: str, record_type: str):
     api_manager = LabelStudioAPIManager(config)
 
     #import tasks
-    label_studio_response = api_manager.import_tasks_into_project(data)
+    label_studio_response = api_manager.export_tasks_into_project(data)
 
     #check import success
     if label_studio_response.status_code == HTTPStatus.CREATED:

api/README.md

@@ -0,0 +1,7 @@
+To spin up a development version of the client, run:
+
+```bash
+fastapi dev main.py
+```
+
+For the client to function properly in a local environment, the local database must be set up. Consult the `README.md` file in the `local_database` directory for further instructions.

api/dependencies.py

@@ -0,0 +1,6 @@
+from core.SourceCollectorCore import SourceCollectorCore
+
+
+def get_core() -> SourceCollectorCore:
+    from api.main import app
+    return app.state.core

api/main.py

@@ -0,0 +1,47 @@
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from api.routes.batch import batch_router
+from api.routes.collector import collector_router
+from api.routes.label_studio import label_studio_router
+from api.routes.root import root_router
+from collector_db.DatabaseClient import DatabaseClient
+from core.CoreLogger import CoreLogger
+from core.SourceCollectorCore import SourceCollectorCore
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Initialize shared dependencies
+    db_client = DatabaseClient()
+    source_collector_core = SourceCollectorCore(
+        core_logger=CoreLogger(
+            db_client=db_client
+        ),
+        db_client=DatabaseClient(),
+    )
+
+    # Pass dependencies into the app state
+    app.state.core = source_collector_core
+
+    # Startup logic
+    yield  # Code here runs before shutdown
+
+    # Shutdown logic (if needed)
+    app.state.core.shutdown()
+    # Clean up resources, close connections, etc.
+    pass
+
+
+app = FastAPI(
+    title="Source Collector API",
+    description="API for collecting data sources",
+    version="0.1.0",
+    lifespan=lifespan
+)
+
+app.include_router(root_router)
+app.include_router(collector_router)
+app.include_router(batch_router)
+app.include_router(label_studio_router)

api/routes/batch.py

@@ -0,0 +1,98 @@
+from typing import Optional
+
+from fastapi import Path, APIRouter
+from fastapi.params import Query, Depends
+
+from api.dependencies import get_core
+from collector_db.DTOs.BatchInfo import BatchInfo
+from collector_manager.enums import CollectorType
+from core.DTOs.GetBatchLogsResponse import GetBatchLogsResponse
+from core.DTOs.GetBatchStatusResponse import GetBatchStatusResponse
+from core.DTOs.GetDuplicatesByBatchResponse import GetDuplicatesByBatchResponse
+from core.DTOs.GetURLsByBatchResponse import GetURLsByBatchResponse
+from core.DTOs.MessageResponse import MessageResponse
+from core.SourceCollectorCore import SourceCollectorCore
+from core.enums import BatchStatus
+from security_manager.SecurityManager import AccessInfo, get_access_info
+
+batch_router = APIRouter(
+    prefix="/batch",
+    tags=["Batch"],
+    responses={404: {"description": "Not found"}},
+)
+
+
+@batch_router.get("")
+def get_batch_status(
+        collector_type: Optional[CollectorType] = Query(
+            description="Filter by collector type",
+            default=None
+        ),
+        status: Optional[BatchStatus] = Query(
+            description="Filter by status",
+            default=None
+        ),
+        page: int = Query(
+            description="The page number",
+            default=1
+        ),
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> GetBatchStatusResponse:
+    """
+    Get the status of recent batches
+    """
+    return core.get_batch_statuses(collector_type=collector_type, status=status, page=page)
+
+
+@batch_router.get("/{batch_id}")
+def get_batch_info(
+        batch_id: int = Path(description="The batch id"),
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> BatchInfo:
+    return core.get_batch_info(batch_id)
+
+@batch_router.get("/{batch_id}/urls")
+def get_urls_by_batch(
+        batch_id: int = Path(description="The batch id"),
+        page: int = Query(
+            description="The page number",
+            default=1
+        ),
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> GetURLsByBatchResponse:
+    return core.get_urls_by_batch(batch_id, page=page)
+
+@batch_router.get("/{batch_id}/duplicates")
+def get_duplicates_by_batch(
+        batch_id: int = Path(description="The batch id"),
+        page: int = Query(
+            description="The page number",
+            default=1
+        ),
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> GetDuplicatesByBatchResponse:
+    return core.get_duplicate_urls_by_batch(batch_id, page=page)
+
+@batch_router.get("/{batch_id}/logs")
+def get_batch_logs(
+        batch_id: int = Path(description="The batch id"),
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> GetBatchLogsResponse:
+    """
+    Retrieve the logs for a recent batch.
+    Note that for later batches, the logs may not be available.
+    """
+    return core.get_batch_logs(batch_id)
+
+@batch_router.post("/{batch_id}/abort")
+def abort_batch(
+        batch_id: int = Path(description="The batch id"),
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> MessageResponse:
+    return core.abort_batch(batch_id)

api/routes/collector.py

@@ -0,0 +1,127 @@
+from typing import Optional
+
+from fastapi import APIRouter, Query
+from fastapi.params import Depends
+
+from api.dependencies import get_core
+from collector_manager.DTOs.ExampleInputDTO import ExampleInputDTO
+from collector_manager.enums import CollectorType
+from core.DTOs.CollectorStartInfo import CollectorStartInfo
+from core.SourceCollectorCore import SourceCollectorCore
+from security_manager.SecurityManager import AccessInfo, get_access_info
+from source_collectors.auto_googler.DTOs import AutoGooglerInputDTO
+from source_collectors.ckan.DTOs import CKANInputDTO
+from source_collectors.common_crawler.DTOs import CommonCrawlerInputDTO
+from source_collectors.muckrock.DTOs import MuckrockCountySearchCollectorInputDTO, \
+    MuckrockAllFOIARequestsCollectorInputDTO, MuckrockSimpleSearchCollectorInputDTO
+
+collector_router = APIRouter(
+    prefix="/collector",
+    tags=["Collector"],
+    responses={404: {"description": "Not found"}},
+)
+
+@collector_router.post("/example")
+async def start_example_collector(
+        dto: ExampleInputDTO,
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> CollectorStartInfo:
+    """
+    Start the example collector
+    """
+    return core.initiate_collector(
+        collector_type=CollectorType.EXAMPLE,
+        dto=dto,
+        user_id=access_info.user_id
+    )
+
+@collector_router.post("/ckan")
+async def start_ckan_collector(
+        dto: CKANInputDTO,
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> CollectorStartInfo:
+    """
+    Start the ckan collector
+    """
+    return core.initiate_collector(
+        collector_type=CollectorType.CKAN,
+        dto=dto,
+        user_id=access_info.user_id
+    )
+
+@collector_router.post("/common-crawler")
+async def start_common_crawler_collector(
+        dto: CommonCrawlerInputDTO,
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> CollectorStartInfo:
+    """
+    Start the common crawler collector
+    """
+    return core.initiate_collector(
+        collector_type=CollectorType.COMMON_CRAWLER,
+        dto=dto,
+        user_id=access_info.user_id
+    )
+
+@collector_router.post("/auto-googler")
+async def start_auto_googler_collector(
+        dto: AutoGooglerInputDTO,
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> CollectorStartInfo:
+    """
+    Start the auto googler collector
+    """
+    return core.initiate_collector(
+        collector_type=CollectorType.AUTO_GOOGLER,
+        dto=dto,
+        user_id=access_info.user_id
+    )
+
+@collector_router.post("/muckrock-simple")
+async def start_muckrock_collector(
+        dto: MuckrockSimpleSearchCollectorInputDTO,
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> CollectorStartInfo:
+    """
+    Start the muckrock collector
+    """
+    return core.initiate_collector(
+        collector_type=CollectorType.MUCKROCK_SIMPLE_SEARCH,
+        dto=dto,
+        user_id=access_info.user_id
+    )
+
+@collector_router.post("/muckrock-county")
+async def start_muckrock_county_collector(
+        dto: MuckrockCountySearchCollectorInputDTO,
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> CollectorStartInfo:
+    """
+    Start the muckrock county level collector
+    """
+    return core.initiate_collector(
+        collector_type=CollectorType.MUCKROCK_COUNTY_SEARCH,
+        dto=dto,
+        user_id=access_info.user_id
+    )
+
+@collector_router.post("/muckrock-all")
+async def start_muckrock_all_foia_collector(
+        dto: MuckrockAllFOIARequestsCollectorInputDTO,
+        core: SourceCollectorCore = Depends(get_core),
+        access_info: AccessInfo = Depends(get_access_info),
+) -> CollectorStartInfo:
+    """
+    Start the muckrock collector for all FOIA requests
+    """
+    return core.initiate_collector(
+        collector_type=CollectorType.MUCKROCK_ALL_SEARCH,
+        dto=dto,
+        user_id=access_info.user_id
+    )