Dev

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

.github/workflows/test_app.yml

@@ -1,27 +1,12 @@
-# This workflow will test the Source Collector App
-# Utilizing the docker-compose file in the root directory
 name: Test Source Collector App
-on: pull_request
 
-#jobs:
-#    build:
-#        runs-on: ubuntu-latest
-#        steps:
-#            - name: Checkout repository
-#              uses: actions/checkout@v4
-#            - name: Run docker-compose
-#              uses: hoverkraft-tech/compose-action@v2.0.1
-#              with:
-#                  compose-file: "docker-compose.yml"
-#            - name: Execute tests in the running service
-#              run: |
-#                  docker ps -a && docker exec data-source-identification-app-1 pytest /app/tests/test_automated
+on: pull_request
 
 jobs:
   container-job:
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    container: python:3.12.8
+    container: python:3.11.9
 
     services:
       postgres:
@@ -34,22 +19,28 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+    env:
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_HOST: postgres
+      POSTGRES_PORT: 5432
+      GOOGLE_API_KEY: TEST
+      GOOGLE_CSE_ID: TEST
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+
       - name: Run tests
         run: |
-          pytest tests/test_automated
-          pytest tests/test_alembic
-        env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_USER: postgres
-          POSTGRES_DB: postgres
-          POSTGRES_HOST: postgres
-          POSTGRES_PORT: 5432
-          GOOGLE_API_KEY: TEST
-          GOOGLE_CSE_ID: TEST
+          uv run pytest tests/test_automated
+          uv run pytest tests/test_alembic

Dockerfile

@@ -1,17 +1,52 @@
 # Dockerfile for Source Collector FastAPI app
 
-FROM python:3.12.8
+FROM python:3.11.9-slim
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 # Set working directory
 WORKDIR /app
 
-# Copy project files
-COPY . .
+COPY pyproject.toml uv.lock ./
 
 # Install dependencies
-RUN pip install --no-cache-dir -r requirements.txt
+ENV UV_PROJECT_ENVIRONMENT="/usr/local/"
+RUN uv sync --locked --no-dev
+# Must call from the root directory because uv does not add playwright to path
+RUN playwright install-deps chromium
+RUN playwright install chromium
+
+
+# Copy project files
+COPY api ./api
+COPY collector_db ./collector_db
+COPY collector_manager ./collector_manager
+COPY core ./core
+COPY html_tag_collector ./html_tag_collector
+COPY hugging_face/url_relevance ./hugging_face/url_relevance
+COPY hugging_face/url_record_type_labeling ./hugging_face/url_record_type_labeling
+COPY hugging_face/HuggingFaceInterface.py ./hugging_face/HuggingFaceInterface.py
+COPY source_collectors ./source_collectors
+COPY util ./util
+COPY alembic.ini ./alembic.ini
+COPY alembic ./alembic
+COPY apply_migrations.py ./apply_migrations.py
+COPY security_manager ./security_manager
+COPY pdap_api_client ./pdap_api_client
+COPY execute.sh ./execute.sh
+COPY .project-root ./.project-root
+
+COPY tests/conftest.py ./tests/conftest.py
+COPY tests/__init__.py ./tests/__init__.py
+COPY tests/test_automated ./tests/test_automated
+COPY tests/test_alembic ./tests/test_alembic
+COPY tests/helpers ./tests/helpers
+
+COPY llm_api_logic ./llm_api_logic
 
 # Expose the application port
 EXPOSE 80
 
-RUN chmod +x execute.sh
+RUN chmod +x execute.sh
+# Use the below for ease of local development, but remove when pushing to GitHub
+# Because there is no .env file in the repository (for security reasons)
+#COPY .env ./.env

ENV.md

@@ -2,17 +2,43 @@ This page provides a full list, with description, of all the environment variabl
 
 Please ensure these are properly defined in a `.env` file in the root directory.
 
-| Name                           | Description                                                                                                                                                                                                                                                       | Example  |
-|--------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
-| `LABEL_STUDIO_ACCESS_TOKEN`    | The access token for the Label Studio API. The access token for the Label Studio API. This can be obtained by logging into Label Studio and navigating to the [user account section](https://app.heartex.com/user/account), where the access token can be copied. | `abc123` |
-| `LABEL_STUDIO_PROJECT_ID`      | The project ID for the Label Studio API. This can be obtained by logging into Label Studio and navigating to the relevant project, where the project id will be in the URL, as in `https://app.heartex.com/projects/58475/`                                       | `58475`  |
-| `LABEL_STUDIO_ORGANIZATION_ID` | The organization ID for the Label Studio API. This can be obtained by logging into Label Studio and navigating to the [Organization section](https://app.heartex.com/organization?page=1), where the organization ID can be copied.                               | `6758`   |
-| `GOOGLE_API_KEY`               | The API key required for accessing the Google Custom Search API                                                                                                                                                                                                   | `abc123` |
-| `GOOGLE_CSE_ID`                | The CSE ID required for accessing the Google Custom Search API                                                                                                                                                                                                    | `abc123` |
-|`POSTGRES_USER`                 | The username for the test database                                                                                                                                                                                                                                | `test_source_collector_user` |
-|`POSTGRES_PASSWORD`             | The password for the test database                                                                                                                                                                                                                                | `HanviliciousHamiltonHilltops` |
-|`POSTGRES_DB`                   | The database name for the test database                                                                                                                                                                                                                           | `source_collector_test_db` |
-|`POSTGRES_HOST`                 | The host for the test database                                                                                                                                                                                                                                    | `127.0.0.1` |
-|`POSTGRES_PORT`                 | The port for the test database                                                                                                                                                                                                                                    | `5432` |
-|`DS_APP_SECRET_KEY`| The secret key used for decoding JWT tokens produced by the Data Sources App. Must match the secret token that is used in the Data Sources App for encoding.                                                                                                      |`abc123`|
-|`DEV`| Set to any value to run the application in development mode.                                                                                                                                                                                                      |`true`|
+| Name                 | Description                                                                                                                                                                                                                                                       | Example                        |
+|----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------|
+| `GOOGLE_API_KEY`     | The API key required for accessing the Google Custom Search API                                                                                                                                                                                                   | `abc123`                       |
+| `GOOGLE_CSE_ID`      | The CSE ID required for accessing the Google Custom Search API                                                                                                                                                                                                    | `abc123`                       |
+|`POSTGRES_USER`       | The username for the test database                                                                                                                                                                                                                                | `test_source_collector_user`   |
+|`POSTGRES_PASSWORD`   | The password for the test database                                                                                                                                                                                                                                | `HanviliciousHamiltonHilltops` |
+|`POSTGRES_DB`         | The database name for the test database                                                                                                                                                                                                                           | `source_collector_test_db`     |
+|`POSTGRES_HOST`       | The host for the test database                                                                                                                                                                                                                                    | `127.0.0.1`                    |
+|`POSTGRES_PORT`       | The port for the test database                                                                                                                                                                                                                                    | `5432`                         |
+|`DS_APP_SECRET_KEY`| The secret key used for decoding JWT tokens produced by the Data Sources App. Must match the secret token `JWT_SECRET_KEY` that is used in the Data Sources App for encoding.                                                                                     | `abc123`                       |
+|`DEV`| Set to any value to run the application in development mode.                                                                                                                                                                                                      | `true`                         |
+|`DEEPSEEK_API_KEY`| The API key required for accessing the DeepSeek API.                                                                                                                                                                                                              | `abc123`                       |
+|`OPENAI_API_KEY`| The API key required for accessing the OpenAI API.                                                                                                                                                                                                                | `abc123`                       |
+|`PDAP_EMAIL`| An email address for accessing the PDAP API.[^1]                                                                                                                                                                                                                  | `abc123@test.com`               |
+|`PDAP_PASSWORD`| A password for accessing the PDAP API.[^1]                                                                                                                                                                                                                        | `abc123`               |
+|`PDAP_API_KEY`| An API key for accessing the PDAP API.                                                                                                                                                                                                                            | `abc123`               |
+|`PDAP_API_URL`| The URL for the PDAP API| `https://data-sources-v2.pdap.dev/api`|
+|`DISCORD_WEBHOOK_URL`| The URL for the Discord webhook used for notifications| `abc123`               |
+
+[^1:] The user account in question will require elevated permissions to access certain endpoints. At a minimum, the user will require the `source_collector` and `db_write` permissions.
+
+## Foreign Data Wrapper (FDW)
+```
+FDW_DATA_SOURCES_HOST=127.0.0.1  # The host of the Data Sources Database, used for FDW setup
+FDW_DATA_SOURCES_PORT=1234                  # The port of the Data Sources Database, used for FDW setup
+FDW_DATA_SOURCES_USER=fdw_user           # The username for the Data Sources Database, used for FDW setup
+FDW_DATA_SOURCES_PASSWORD=password          # The password for the Data Sources Database, used for FDW setup
+FDW_DATA_SOURCES_DB=db_name                  # The database name for the Data Sources Database, used for FDW setup
+
+```
+
+## Data Dumper
+
+```
+PROD_DATA_SOURCES_HOST=127.0.0.1  # The host of the production Data Sources Database, used for Data Dumper
+PROD_DATA_SOURCES_PORT=1234                  # The port of the production Data Sources Database, used for Data Dumper
+PROD_DATA_SOURCES_USER=dump_user           # The username for the production Data Sources Database, used for Data Dumper
+PROD_DATA_SOURCES_PASSWORD=password          # The password for the production Data Sources Database, used for Data Dumper
+PROD_DATA_SOURCES_DB=db_name                  # The database name for the production Data Sources Database, used for Data Dumper
+```