diff --git a/.github/workflows/test_app.yml b/.github/workflows/test_app.yml index 64bf664e..64164dc3 100644 --- a/.github/workflows/test_app.yml +++ b/.github/workflows/test_app.yml @@ -42,5 +42,5 @@ jobs: - name: Run tests run: | - uv run pytest tests/test_automated - uv run pytest tests/test_alembic + uv run pytest tests/automated + uv run pytest tests/alembic diff --git a/Dockerfile b/Dockerfile index 258a3a31..5ba90408 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN playwright install chromium # Copy project files COPY api ./api -COPY collector_db ./collector_db +COPY db ./collector_db COPY collector_manager ./collector_manager COPY core ./core COPY html_tag_collector ./html_tag_collector @@ -32,12 +32,6 @@ COPY pdap_api_client ./pdap_api_client COPY execute.sh ./execute.sh COPY .project-root ./.project-root -COPY tests/conftest.py ./tests/conftest.py -COPY tests/__init__.py ./tests/__init__.py -COPY tests/test_automated ./tests/test_automated -COPY tests/test_alembic ./tests/test_alembic -COPY tests/helpers ./tests/helpers - COPY llm_api_logic ./llm_api_logic # Expose the application port diff --git a/README.md b/README.md index 701561db..794b968e 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ To access the API documentation, visit `http://{host}:8000/docs`. To run tests on the container, run: ```bash -docker exec data-source-identification-app-1 pytest /app/tests/test_automated +docker exec data-source-identification-app-1 pytest /app/tests/automated ``` Be sure to inspect the `docker-compose.yml` file in the root directory -- some environment variables are dependant upon the Operating System you are using. diff --git a/alembic/env.py b/alembic/env.py index 7eaa1a8b..3ba2f117 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -5,8 +5,8 @@ from sqlalchemy import engine_from_config from sqlalchemy import pool -from collector_db.helper_functions import get_postgres_connection_string -from collector_db.models import Base +from db.helper_functions import get_postgres_connection_string +from db.models import Base # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/alembic/versions/072b32a45b1c_add_task_tables_and_linking_logic.py b/alembic/versions/072b32a45b1c_add_task_tables_and_linking_logic.py index f408396f..8661f524 100644 --- a/alembic/versions/072b32a45b1c_add_task_tables_and_linking_logic.py +++ b/alembic/versions/072b32a45b1c_add_task_tables_and_linking_logic.py @@ -10,7 +10,7 @@ from alembic import op import sqlalchemy as sa -from collector_db.enums import PGEnum +from db.enums import PGEnum # revision identifiers, used by Alembic. revision: str = '072b32a45b1c' diff --git a/alembic/versions/19bf57df581a_add_url_agency_suggestions.py b/alembic/versions/19bf57df581a_add_url_agency_suggestions.py index 608fcd1b..f735e271 100644 --- a/alembic/versions/19bf57df581a_add_url_agency_suggestions.py +++ b/alembic/versions/19bf57df581a_add_url_agency_suggestions.py @@ -9,7 +9,7 @@ from alembic import op import sqlalchemy as sa -from collector_db.enums import PGEnum +from db.enums import PGEnum # revision identifiers, used by Alembic. revision: str = '19bf57df581a' down_revision: Union[str, None] = '072b32a45b1c' diff --git a/alembic/versions/9afd8a5633c9_create_htmlcontent_and_rooturl_tables.py b/alembic/versions/9afd8a5633c9_create_htmlcontent_and_rooturl_tables.py index 81401c06..bc26bf30 100644 --- a/alembic/versions/9afd8a5633c9_create_htmlcontent_and_rooturl_tables.py +++ b/alembic/versions/9afd8a5633c9_create_htmlcontent_and_rooturl_tables.py @@ -10,7 +10,7 @@ import sqlalchemy as sa from alembic import op -from collector_db.enums import PGEnum +from db.enums import PGEnum # revision identifiers, used by Alembic. revision: str = '9afd8a5633c9' diff --git a/alembic/versions/d7eb670edaf0_revise_agency_identification_logic.py b/alembic/versions/d7eb670edaf0_revise_agency_identification_logic.py index 2bb7c157..16611eee 100644 --- a/alembic/versions/d7eb670edaf0_revise_agency_identification_logic.py +++ b/alembic/versions/d7eb670edaf0_revise_agency_identification_logic.py @@ -10,7 +10,7 @@ from alembic import op import sqlalchemy as sa -from collector_db.enums import PGEnum +from db.enums import PGEnum # revision identifiers, used by Alembic. revision: str = 'd7eb670edaf0' diff --git a/api/main.py b/api/main.py index 414f7223..fd82df85 100644 --- a/api/main.py +++ b/api/main.py @@ -14,8 +14,8 @@ from api.routes.search import search_router from api.routes.task import task_router from api.routes.url import url_router -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DatabaseClient import DatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DatabaseClient import DatabaseClient from collector_manager.AsyncCollectorManager import AsyncCollectorManager from core.AsyncCore import AsyncCore from core.AsyncCoreLogger import AsyncCoreLogger diff --git a/api/routes/batch.py b/api/routes/batch.py index 7ba0a2a4..4ca38c55 100644 --- a/api/routes/batch.py +++ b/api/routes/batch.py @@ -4,7 +4,7 @@ from fastapi.params import Query, Depends from api.dependencies import get_core, get_async_core -from collector_db.DTOs.BatchInfo import BatchInfo +from db.DTOs.BatchInfo import BatchInfo from collector_manager.enums import CollectorType from core.AsyncCore import AsyncCore from core.DTOs.GetBatchLogsResponse import GetBatchLogsResponse diff --git a/api/routes/task.py b/api/routes/task.py index 44971959..a99598f7 100644 --- a/api/routes/task.py +++ b/api/routes/task.py @@ -3,9 +3,9 @@ from fastapi import APIRouter, Depends, Query, Path from api.dependencies import get_async_core -from collector_db.DTOs.GetTaskStatusResponseInfo import GetTaskStatusResponseInfo -from collector_db.DTOs.TaskInfo import TaskInfo -from collector_db.enums import TaskType +from db.DTOs.GetTaskStatusResponseInfo import GetTaskStatusResponseInfo +from db.DTOs.TaskInfo import TaskInfo +from db.enums import TaskType from core.AsyncCore import AsyncCore from core.enums import BatchStatus from security_manager.SecurityManager import AccessInfo, get_access_info diff --git a/apply_migrations.py b/apply_migrations.py index 183e7d11..2dc207ce 100644 --- a/apply_migrations.py +++ b/apply_migrations.py @@ -1,7 +1,7 @@ from alembic import command from alembic.config import Config -from collector_db.helper_functions import get_postgres_connection_string +from db.helper_functions import get_postgres_connection_string def apply_migrations(): print("Applying migrations...") diff --git a/collector_manager/AsyncCollectorBase.py b/collector_manager/AsyncCollectorBase.py index a842a9c0..94361ed4 100644 --- a/collector_manager/AsyncCollectorBase.py +++ b/collector_manager/AsyncCollectorBase.py @@ -6,9 +6,9 @@ from pydantic import BaseModel -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo -from collector_db.DTOs.LogInfo import LogInfo +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.LogInfo import LogInfo from collector_manager.enums import CollectorType from core.AsyncCoreLogger import AsyncCoreLogger from core.FunctionTrigger import FunctionTrigger diff --git a/collector_manager/AsyncCollectorManager.py b/collector_manager/AsyncCollectorManager.py index 1851bfc9..bfb7beef 100644 --- a/collector_manager/AsyncCollectorManager.py +++ b/collector_manager/AsyncCollectorManager.py @@ -5,7 +5,7 @@ from fastapi import HTTPException from pydantic import BaseModel -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from collector_manager.AsyncCollectorBase import AsyncCollectorBase from collector_manager.CollectorManager import InvalidCollectorError from collector_manager.collector_mapping import COLLECTOR_MAPPING diff --git a/core/AsyncCore.py b/core/AsyncCore.py index 651cd3ba..1d61557e 100644 --- a/core/AsyncCore.py +++ b/core/AsyncCore.py @@ -3,10 +3,10 @@ from pydantic import BaseModel from sqlalchemy.exc import IntegrityError -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.BatchInfo import BatchInfo -from collector_db.DTOs.GetTaskStatusResponseInfo import GetTaskStatusResponseInfo -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.BatchInfo import BatchInfo +from db.DTOs.GetTaskStatusResponseInfo import GetTaskStatusResponseInfo +from db.enums import TaskType from collector_manager.AsyncCollectorManager import AsyncCollectorManager from collector_manager.enums import CollectorType from core.DTOs.AllAnnotationPostInfo import AllAnnotationPostInfo diff --git a/core/AsyncCoreLogger.py b/core/AsyncCoreLogger.py index 70ca06aa..67bb5dc9 100644 --- a/core/AsyncCoreLogger.py +++ b/core/AsyncCoreLogger.py @@ -1,7 +1,7 @@ import asyncio -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.LogInfo import LogInfo +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.LogInfo import LogInfo class AsyncCoreLogger: diff --git a/core/DTOs/CollectionLifecycleInfo.py b/core/DTOs/CollectionLifecycleInfo.py index dc1b538f..925dee93 100644 --- a/core/DTOs/CollectionLifecycleInfo.py +++ b/core/DTOs/CollectionLifecycleInfo.py @@ -1,7 +1,7 @@ from pydantic import BaseModel -from collector_db.DTOs.DuplicateInfo import DuplicateInfo -from collector_db.DTOs.URLMapping import URLMapping +from db.DTOs.DuplicateInfo import DuplicateInfo +from db.DTOs.URLMapping import URLMapping class CollectionLifecycleInfo(BaseModel): diff --git a/core/DTOs/GetBatchLogsResponse.py b/core/DTOs/GetBatchLogsResponse.py index 9db96d91..adcc3be9 100644 --- a/core/DTOs/GetBatchLogsResponse.py +++ b/core/DTOs/GetBatchLogsResponse.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from collector_db.DTOs.LogInfo import LogOutputInfo +from db.DTOs.LogInfo import LogOutputInfo class GetBatchLogsResponse(BaseModel): diff --git a/core/DTOs/GetBatchStatusResponse.py b/core/DTOs/GetBatchStatusResponse.py index 2624f0c3..d1a02dc7 100644 --- a/core/DTOs/GetBatchStatusResponse.py +++ b/core/DTOs/GetBatchStatusResponse.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from collector_db.DTOs.BatchInfo import BatchInfo +from db.DTOs.BatchInfo import BatchInfo class GetBatchStatusResponse(BaseModel): diff --git a/core/DTOs/GetDuplicatesByBatchResponse.py b/core/DTOs/GetDuplicatesByBatchResponse.py index b2915d61..68a6dd4b 100644 --- a/core/DTOs/GetDuplicatesByBatchResponse.py +++ b/core/DTOs/GetDuplicatesByBatchResponse.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from collector_db.DTOs.DuplicateInfo import DuplicateInfo +from db.DTOs.DuplicateInfo import DuplicateInfo class GetDuplicatesByBatchResponse(BaseModel): diff --git a/core/DTOs/GetNextRecordTypeAnnotationResponseInfo.py b/core/DTOs/GetNextRecordTypeAnnotationResponseInfo.py index 4280e00d..e1784409 100644 --- a/core/DTOs/GetNextRecordTypeAnnotationResponseInfo.py +++ b/core/DTOs/GetNextRecordTypeAnnotationResponseInfo.py @@ -2,7 +2,7 @@ from pydantic import Field, BaseModel -from collector_db.DTOs.URLMapping import URLMapping +from db.DTOs.URLMapping import URLMapping from core.enums import RecordType from html_tag_collector.DataClassTags import ResponseHTMLInfo diff --git a/core/DTOs/GetNextRelevanceAnnotationResponseInfo.py b/core/DTOs/GetNextRelevanceAnnotationResponseInfo.py index 61cb35a5..af586395 100644 --- a/core/DTOs/GetNextRelevanceAnnotationResponseInfo.py +++ b/core/DTOs/GetNextRelevanceAnnotationResponseInfo.py @@ -2,7 +2,7 @@ from pydantic import BaseModel, Field -from collector_db.DTOs.URLMapping import URLMapping +from db.DTOs.URLMapping import URLMapping from core.DTOs.ResponseURLInfo import ResponseURLInfo from html_tag_collector.DataClassTags import ResponseHTMLInfo diff --git a/core/DTOs/GetTasksResponse.py b/core/DTOs/GetTasksResponse.py index 42b3d954..670ce8d3 100644 --- a/core/DTOs/GetTasksResponse.py +++ b/core/DTOs/GetTasksResponse.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from collector_db.enums import TaskType +from db.enums import TaskType from core.enums import BatchStatus diff --git a/core/DTOs/GetURLsByBatchResponse.py b/core/DTOs/GetURLsByBatchResponse.py index ce4ed5f8..c737d720 100644 --- a/core/DTOs/GetURLsByBatchResponse.py +++ b/core/DTOs/GetURLsByBatchResponse.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo class GetURLsByBatchResponse(BaseModel): diff --git a/core/DTOs/GetURLsResponseInfo.py b/core/DTOs/GetURLsResponseInfo.py index 162e92b5..a924d5aa 100644 --- a/core/DTOs/GetURLsResponseInfo.py +++ b/core/DTOs/GetURLsResponseInfo.py @@ -3,7 +3,7 @@ from pydantic import BaseModel -from collector_db.enums import URLMetadataAttributeType, ValidationStatus, ValidationSource +from db.enums import URLMetadataAttributeType, ValidationStatus, ValidationSource from collector_manager.enums import URLStatus class GetURLsResponseErrorInfo(BaseModel): diff --git a/core/DTOs/task_data_objects/URLRecordTypeTDO.py b/core/DTOs/task_data_objects/URLRecordTypeTDO.py index 34bbc233..03215f71 100644 --- a/core/DTOs/task_data_objects/URLRecordTypeTDO.py +++ b/core/DTOs/task_data_objects/URLRecordTypeTDO.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from collector_db.DTOs.URLWithHTML import URLWithHTML +from db.DTOs.URLWithHTML import URLWithHTML from core.enums import RecordType diff --git a/core/DTOs/task_data_objects/UrlHtmlTDO.py b/core/DTOs/task_data_objects/UrlHtmlTDO.py index ca5b8a50..96c44778 100644 --- a/core/DTOs/task_data_objects/UrlHtmlTDO.py +++ b/core/DTOs/task_data_objects/UrlHtmlTDO.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo from html_tag_collector.DataClassTags import ResponseHTMLInfo from html_tag_collector.URLRequestInterface import URLResponseInfo diff --git a/core/SourceCollectorCore.py b/core/SourceCollectorCore.py index a4699bf6..6b9822ae 100644 --- a/core/SourceCollectorCore.py +++ b/core/SourceCollectorCore.py @@ -1,7 +1,7 @@ from typing import Optional, Any -from collector_db.DatabaseClient import DatabaseClient +from db.DatabaseClient import DatabaseClient from core.enums import BatchStatus diff --git a/core/TaskManager.py b/core/TaskManager.py index 4dd92450..4424ec1c 100644 --- a/core/TaskManager.py +++ b/core/TaskManager.py @@ -3,9 +3,9 @@ from core.classes.task_operators.URL404ProbeTaskOperator import URL404ProbeTaskOperator from core.classes.task_operators.URLDuplicateTaskOperator import URLDuplicateTaskOperator from source_collectors.muckrock.MuckrockAPIInterface import MuckrockAPIInterface -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.TaskInfo import TaskInfo -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.TaskInfo import TaskInfo +from db.enums import TaskType from core.DTOs.GetTasksResponse import GetTasksResponse from core.DTOs.TaskOperatorRunInfo import TaskOperatorRunInfo, TaskOperatorOutcome from core.FunctionTrigger import FunctionTrigger diff --git a/core/classes/HTMLContentInfoGetter.py b/core/classes/HTMLContentInfoGetter.py index c9d7af61..b9e0b7e1 100644 --- a/core/classes/HTMLContentInfoGetter.py +++ b/core/classes/HTMLContentInfoGetter.py @@ -1,4 +1,4 @@ -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType from html_tag_collector.DataClassTags import ResponseHTMLInfo diff --git a/core/classes/task_operators/AgencyIdentificationTaskOperator.py b/core/classes/task_operators/AgencyIdentificationTaskOperator.py index b6e53955..259ddff9 100644 --- a/core/classes/task_operators/AgencyIdentificationTaskOperator.py +++ b/core/classes/task_operators/AgencyIdentificationTaskOperator.py @@ -1,9 +1,9 @@ from aiohttp import ClientSession from source_collectors.muckrock.MuckrockAPIInterface import MuckrockAPIInterface -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.enums import TaskType from collector_manager.enums import CollectorType from core.DTOs.URLAgencySuggestionInfo import URLAgencySuggestionInfo from core.DTOs.task_data_objects.AgencyIdentificationTDO import AgencyIdentificationTDO diff --git a/core/classes/task_operators/SubmitApprovedURLTaskOperator.py b/core/classes/task_operators/SubmitApprovedURLTaskOperator.py index 86e0229e..e3a4eab2 100644 --- a/core/classes/task_operators/SubmitApprovedURLTaskOperator.py +++ b/core/classes/task_operators/SubmitApprovedURLTaskOperator.py @@ -1,6 +1,6 @@ -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.enums import TaskType from core.DTOs.task_data_objects.SubmitApprovedURLTDO import SubmitApprovedURLTDO from core.classes.task_operators.TaskOperatorBase import TaskOperatorBase from pdap_api_client.PDAPClient import PDAPClient diff --git a/core/classes/task_operators/TaskOperatorBase.py b/core/classes/task_operators/TaskOperatorBase.py index e7c87dac..df12f362 100644 --- a/core/classes/task_operators/TaskOperatorBase.py +++ b/core/classes/task_operators/TaskOperatorBase.py @@ -1,7 +1,7 @@ import traceback from abc import ABC, abstractmethod -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.enums import TaskType from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome, TaskOperatorRunInfo from core.enums import BatchStatus diff --git a/core/classes/task_operators/URL404ProbeTaskOperator.py b/core/classes/task_operators/URL404ProbeTaskOperator.py index 08f513f8..536fea23 100644 --- a/core/classes/task_operators/URL404ProbeTaskOperator.py +++ b/core/classes/task_operators/URL404ProbeTaskOperator.py @@ -2,8 +2,8 @@ from pydantic import BaseModel -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.enums import TaskType from core.DTOs.task_data_objects.URL404ProbeTDO import URL404ProbeTDO from core.classes.task_operators.TaskOperatorBase import TaskOperatorBase from html_tag_collector.URLRequestInterface import URLRequestInterface diff --git a/core/classes/task_operators/URLDuplicateTaskOperator.py b/core/classes/task_operators/URLDuplicateTaskOperator.py index 060c07c7..09ab35ce 100644 --- a/core/classes/task_operators/URLDuplicateTaskOperator.py +++ b/core/classes/task_operators/URLDuplicateTaskOperator.py @@ -2,8 +2,8 @@ from aiohttp import ClientResponseError -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.enums import TaskType from core.DTOs.task_data_objects.URLDuplicateTDO import URLDuplicateTDO from core.classes.task_operators.TaskOperatorBase import TaskOperatorBase from pdap_api_client.PDAPClient import PDAPClient diff --git a/core/classes/task_operators/URLHTMLTaskOperator.py b/core/classes/task_operators/URLHTMLTaskOperator.py index 17f383c7..340c386b 100644 --- a/core/classes/task_operators/URLHTMLTaskOperator.py +++ b/core/classes/task_operators/URLHTMLTaskOperator.py @@ -1,9 +1,9 @@ from http import HTTPStatus -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.DTOs.URLInfo import URLInfo -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.DTOs.URLInfo import URLInfo +from db.enums import TaskType from core.DTOs.task_data_objects.UrlHtmlTDO import UrlHtmlTDO from core.classes.HTMLContentInfoGetter import HTMLContentInfoGetter from core.classes.task_operators.TaskOperatorBase import TaskOperatorBase diff --git a/core/classes/task_operators/URLMiscellaneousMetadataTaskOperator.py b/core/classes/task_operators/URLMiscellaneousMetadataTaskOperator.py index 68a3a243..bf9eef04 100644 --- a/core/classes/task_operators/URLMiscellaneousMetadataTaskOperator.py +++ b/core/classes/task_operators/URLMiscellaneousMetadataTaskOperator.py @@ -1,8 +1,8 @@ from typing import Optional -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.enums import TaskType from collector_manager.enums import CollectorType from core.DTOs.task_data_objects.URLMiscellaneousMetadataTDO import URLMiscellaneousMetadataTDO from core.classes.task_operators.TaskOperatorBase import TaskOperatorBase diff --git a/core/classes/task_operators/URLRecordTypeTaskOperator.py b/core/classes/task_operators/URLRecordTypeTaskOperator.py index ab1f1f08..0f080a03 100644 --- a/core/classes/task_operators/URLRecordTypeTaskOperator.py +++ b/core/classes/task_operators/URLRecordTypeTaskOperator.py @@ -1,6 +1,6 @@ -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.enums import TaskType from core.DTOs.task_data_objects.URLRecordTypeTDO import URLRecordTypeTDO from core.classes.task_operators.TaskOperatorBase import TaskOperatorBase from core.enums import RecordType diff --git a/core/preprocessors/AutoGooglerPreprocessor.py b/core/preprocessors/AutoGooglerPreprocessor.py index 4124e04b..ebbf4474 100644 --- a/core/preprocessors/AutoGooglerPreprocessor.py +++ b/core/preprocessors/AutoGooglerPreprocessor.py @@ -1,6 +1,6 @@ from typing import List -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo from core.preprocessors.PreprocessorBase import PreprocessorBase diff --git a/core/preprocessors/CKANPreprocessor.py b/core/preprocessors/CKANPreprocessor.py index 65017300..62a550a1 100644 --- a/core/preprocessors/CKANPreprocessor.py +++ b/core/preprocessors/CKANPreprocessor.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import List -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo class CKANPreprocessor: diff --git a/core/preprocessors/CommonCrawlerPreprocessor.py b/core/preprocessors/CommonCrawlerPreprocessor.py index dbf1b885..018d9bfb 100644 --- a/core/preprocessors/CommonCrawlerPreprocessor.py +++ b/core/preprocessors/CommonCrawlerPreprocessor.py @@ -1,6 +1,6 @@ from typing import List -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo from core.preprocessors.PreprocessorBase import PreprocessorBase diff --git a/core/preprocessors/ExamplePreprocessor.py b/core/preprocessors/ExamplePreprocessor.py index 46979039..41f3b57e 100644 --- a/core/preprocessors/ExamplePreprocessor.py +++ b/core/preprocessors/ExamplePreprocessor.py @@ -1,6 +1,6 @@ from typing import List -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo from collector_manager.DTOs.ExampleOutputDTO import ExampleOutputDTO from core.preprocessors.PreprocessorBase import PreprocessorBase diff --git a/core/preprocessors/MuckrockPreprocessor.py b/core/preprocessors/MuckrockPreprocessor.py index b0be0e36..04ba221b 100644 --- a/core/preprocessors/MuckrockPreprocessor.py +++ b/core/preprocessors/MuckrockPreprocessor.py @@ -1,6 +1,6 @@ from typing import List -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo from core.preprocessors.PreprocessorBase import PreprocessorBase diff --git a/core/preprocessors/PreprocessorBase.py b/core/preprocessors/PreprocessorBase.py index b39264b0..6f44f8ae 100644 --- a/core/preprocessors/PreprocessorBase.py +++ b/core/preprocessors/PreprocessorBase.py @@ -2,7 +2,7 @@ from abc import ABC from typing import List -from collector_db.DTOs.URLInfo import URLInfo +from db.DTOs.URLInfo import URLInfo class PreprocessorBase(ABC): diff --git a/collector_db/AsyncDatabaseClient.py b/db/AsyncDatabaseClient.py similarity index 98% rename from collector_db/AsyncDatabaseClient.py rename to db/AsyncDatabaseClient.py index 81ab8de2..a539d3b2 100644 --- a/collector_db/AsyncDatabaseClient.py +++ b/db/AsyncDatabaseClient.py @@ -12,21 +12,21 @@ from sqlalchemy.sql.functions import coalesce from starlette import status -from collector_db.ConfigManager import ConfigManager -from collector_db.DTOConverter import DTOConverter -from collector_db.DTOs.BatchInfo import BatchInfo -from collector_db.DTOs.DuplicateInfo import DuplicateInsertInfo, DuplicateInfo -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo -from collector_db.DTOs.LogInfo import LogInfo, LogOutputInfo -from collector_db.DTOs.TaskInfo import TaskInfo -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType -from collector_db.DTOs.URLInfo import URLInfo -from collector_db.DTOs.URLMapping import URLMapping -from collector_db.StatementComposer import StatementComposer -from collector_db.constants import PLACEHOLDER_AGENCY_NAME -from collector_db.enums import TaskType -from collector_db.models import URL, URLErrorInfo, URLHTMLContent, Base, \ +from db.ConfigManager import ConfigManager +from db.DTOConverter import DTOConverter +from db.DTOs.BatchInfo import BatchInfo +from db.DTOs.DuplicateInfo import DuplicateInsertInfo, DuplicateInfo +from db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.LogInfo import LogInfo, LogOutputInfo +from db.DTOs.TaskInfo import TaskInfo +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType +from db.DTOs.URLInfo import URLInfo +from db.DTOs.URLMapping import URLMapping +from db.StatementComposer import StatementComposer +from db.constants import PLACEHOLDER_AGENCY_NAME +from db.enums import TaskType +from db.models import URL, URLErrorInfo, URLHTMLContent, Base, \ RootURL, Task, TaskError, LinkTaskURL, Batch, Agency, AutomatedUrlAgencySuggestion, \ UserUrlAgencySuggestion, AutoRelevantSuggestion, AutoRecordTypeSuggestion, UserRelevantSuggestion, \ UserRecordTypeSuggestion, ReviewingUserURL, URLOptionalDataSourceMetadata, ConfirmedURLAgency, Duplicate, Log, \ diff --git a/collector_db/ConfigManager.py b/db/ConfigManager.py similarity index 100% rename from collector_db/ConfigManager.py rename to db/ConfigManager.py diff --git a/collector_db/DTOConverter.py b/db/DTOConverter.py similarity index 95% rename from collector_db/DTOConverter.py rename to db/DTOConverter.py index d06ac6de..d95935d4 100644 --- a/collector_db/DTOConverter.py +++ b/db/DTOConverter.py @@ -1,9 +1,9 @@ from typing import Optional -from collector_db.DTOs.URLHTMLContentInfo import HTMLContentType, URLHTMLContentInfo -from collector_db.DTOs.URLInfo import URLInfo -from collector_db.DTOs.URLWithHTML import URLWithHTML -from collector_db.models import AutomatedUrlAgencySuggestion, UserUrlAgencySuggestion, URLHTMLContent, URL, Agency, \ +from db.DTOs.URLHTMLContentInfo import HTMLContentType, URLHTMLContentInfo +from db.DTOs.URLInfo import URLInfo +from db.DTOs.URLWithHTML import URLWithHTML +from db.models import AutomatedUrlAgencySuggestion, UserUrlAgencySuggestion, URLHTMLContent, URL, Agency, \ AutoRecordTypeSuggestion, UserRecordTypeSuggestion, UserRelevantSuggestion, AutoRelevantSuggestion, \ ConfirmedURLAgency from core.DTOs.GetNextURLForAgencyAnnotationResponse import GetNextURLForAgencyAgencyInfo diff --git a/collector_db/DTOs/BatchInfo.py b/db/DTOs/BatchInfo.py similarity index 100% rename from collector_db/DTOs/BatchInfo.py rename to db/DTOs/BatchInfo.py diff --git a/collector_db/DTOs/DuplicateInfo.py b/db/DTOs/DuplicateInfo.py similarity index 100% rename from collector_db/DTOs/DuplicateInfo.py rename to db/DTOs/DuplicateInfo.py diff --git a/collector_db/DTOs/GetTaskStatusResponseInfo.py b/db/DTOs/GetTaskStatusResponseInfo.py similarity index 71% rename from collector_db/DTOs/GetTaskStatusResponseInfo.py rename to db/DTOs/GetTaskStatusResponseInfo.py index f6a8d5fc..df44fd73 100644 --- a/collector_db/DTOs/GetTaskStatusResponseInfo.py +++ b/db/DTOs/GetTaskStatusResponseInfo.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from collector_db.enums import TaskType +from db.enums import TaskType class GetTaskStatusResponseInfo(BaseModel): diff --git a/collector_db/DTOs/InsertURLsInfo.py b/db/DTOs/InsertURLsInfo.py similarity index 79% rename from collector_db/DTOs/InsertURLsInfo.py rename to db/DTOs/InsertURLsInfo.py index da2ee39a..b7cbc924 100644 --- a/collector_db/DTOs/InsertURLsInfo.py +++ b/db/DTOs/InsertURLsInfo.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from collector_db.DTOs.URLMapping import URLMapping +from db.DTOs.URLMapping import URLMapping class InsertURLsInfo(BaseModel): diff --git a/collector_db/DTOs/LogInfo.py b/db/DTOs/LogInfo.py similarity index 100% rename from collector_db/DTOs/LogInfo.py rename to db/DTOs/LogInfo.py diff --git a/collector_db/DTOs/MetadataAnnotationInfo.py b/db/DTOs/MetadataAnnotationInfo.py similarity index 100% rename from collector_db/DTOs/MetadataAnnotationInfo.py rename to db/DTOs/MetadataAnnotationInfo.py diff --git a/collector_db/DTOs/README.md b/db/DTOs/README.md similarity index 100% rename from collector_db/DTOs/README.md rename to db/DTOs/README.md diff --git a/collector_db/DTOs/TaskInfo.py b/db/DTOs/TaskInfo.py similarity index 68% rename from collector_db/DTOs/TaskInfo.py rename to db/DTOs/TaskInfo.py index e8d8090d..feae2666 100644 --- a/collector_db/DTOs/TaskInfo.py +++ b/db/DTOs/TaskInfo.py @@ -3,9 +3,9 @@ from pydantic import BaseModel -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.DTOs.URLInfo import URLInfo -from collector_db.enums import TaskType +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.DTOs.URLInfo import URLInfo +from db.enums import TaskType from core.enums import BatchStatus diff --git a/collector_db/DTOs/URLAnnotationInfo.py b/db/DTOs/URLAnnotationInfo.py similarity index 71% rename from collector_db/DTOs/URLAnnotationInfo.py rename to db/DTOs/URLAnnotationInfo.py index 844b226d..e83e4752 100644 --- a/collector_db/DTOs/URLAnnotationInfo.py +++ b/db/DTOs/URLAnnotationInfo.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo class URLAnnotationInfo(BaseModel): diff --git a/collector_db/DTOs/URLErrorInfos.py b/db/DTOs/URLErrorInfos.py similarity index 100% rename from collector_db/DTOs/URLErrorInfos.py rename to db/DTOs/URLErrorInfos.py diff --git a/collector_db/DTOs/URLHTMLContentInfo.py b/db/DTOs/URLHTMLContentInfo.py similarity index 100% rename from collector_db/DTOs/URLHTMLContentInfo.py rename to db/DTOs/URLHTMLContentInfo.py diff --git a/collector_db/DTOs/URLInfo.py b/db/DTOs/URLInfo.py similarity index 100% rename from collector_db/DTOs/URLInfo.py rename to db/DTOs/URLInfo.py diff --git a/collector_db/DTOs/URLMapping.py b/db/DTOs/URLMapping.py similarity index 100% rename from collector_db/DTOs/URLMapping.py rename to db/DTOs/URLMapping.py diff --git a/collector_db/DTOs/URLMetadataInfo.py b/db/DTOs/URLMetadataInfo.py similarity index 86% rename from collector_db/DTOs/URLMetadataInfo.py rename to db/DTOs/URLMetadataInfo.py index 461d16e9..27431a99 100644 --- a/collector_db/DTOs/URLMetadataInfo.py +++ b/db/DTOs/URLMetadataInfo.py @@ -3,7 +3,7 @@ from pydantic import BaseModel -from collector_db.enums import URLMetadataAttributeType, ValidationStatus, ValidationSource +from db.enums import URLMetadataAttributeType, ValidationStatus, ValidationSource class URLMetadataInfo(BaseModel): diff --git a/collector_db/DTOs/URLRelevancyInfo.py b/db/DTOs/URLRelevancyInfo.py similarity index 100% rename from collector_db/DTOs/URLRelevancyInfo.py rename to db/DTOs/URLRelevancyInfo.py diff --git a/collector_db/DTOs/URLWithHTML.py b/db/DTOs/URLWithHTML.py similarity index 66% rename from collector_db/DTOs/URLWithHTML.py rename to db/DTOs/URLWithHTML.py index 59ba63ee..53c77b82 100644 --- a/collector_db/DTOs/URLWithHTML.py +++ b/db/DTOs/URLWithHTML.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo class URLWithHTML(BaseModel): diff --git a/collector_db/DTOs/__init__.py b/db/DTOs/__init__.py similarity index 100% rename from collector_db/DTOs/__init__.py rename to db/DTOs/__init__.py diff --git a/collector_db/DatabaseClient.py b/db/DatabaseClient.py similarity index 94% rename from collector_db/DatabaseClient.py rename to db/DatabaseClient.py index 8bd8105f..030e2db6 100644 --- a/collector_db/DatabaseClient.py +++ b/db/DatabaseClient.py @@ -6,14 +6,14 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import sessionmaker, scoped_session, Session -from collector_db.ConfigManager import ConfigManager -from collector_db.DTOs.BatchInfo import BatchInfo -from collector_db.DTOs.DuplicateInfo import DuplicateInsertInfo -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo -from collector_db.DTOs.LogInfo import LogInfo -from collector_db.DTOs.URLInfo import URLInfo -from collector_db.DTOs.URLMapping import URLMapping -from collector_db.models import Base, Batch, URL, Log, Duplicate, URLDataSource +from db.ConfigManager import ConfigManager +from db.DTOs.BatchInfo import BatchInfo +from db.DTOs.DuplicateInfo import DuplicateInsertInfo +from db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.LogInfo import LogInfo +from db.DTOs.URLInfo import URLInfo +from db.DTOs.URLMapping import URLMapping +from db.models import Base, Batch, URL, Log, Duplicate, URLDataSource from collector_manager.enums import CollectorType, URLStatus from core.DTOs.ManualBatchInputDTO import ManualBatchInputDTO from core.DTOs.ManualBatchResponseDTO import ManualBatchResponseDTO diff --git a/collector_db/README.md b/db/README.md similarity index 100% rename from collector_db/README.md rename to db/README.md diff --git a/collector_db/StatementComposer.py b/db/StatementComposer.py similarity index 94% rename from collector_db/StatementComposer.py rename to db/StatementComposer.py index 2ea33c5f..73121e51 100644 --- a/collector_db/StatementComposer.py +++ b/db/StatementComposer.py @@ -3,8 +3,8 @@ from sqlalchemy import Select, select, exists, Table, func, Subquery, and_, not_, ColumnElement, case, literal, CTE from sqlalchemy.orm import aliased -from collector_db.enums import URLMetadataAttributeType, ValidationStatus, TaskType -from collector_db.models import URL, URLHTMLContent, AutomatedUrlAgencySuggestion, URLOptionalDataSourceMetadata, Batch, \ +from db.enums import URLMetadataAttributeType, ValidationStatus, TaskType +from db.models import URL, URLHTMLContent, AutomatedUrlAgencySuggestion, URLOptionalDataSourceMetadata, Batch, \ ConfirmedURLAgency, LinkTaskURL, Task, UserUrlAgencySuggestion, UserRecordTypeSuggestion, UserRelevantSuggestion, \ AutoRecordTypeSuggestion, AutoRelevantSuggestion, ReviewingUserURL from collector_manager.enums import URLStatus, CollectorType diff --git a/collector_db/__init__.py b/db/__init__.py similarity index 100% rename from collector_db/__init__.py rename to db/__init__.py diff --git a/collector_db/constants.py b/db/constants.py similarity index 100% rename from collector_db/constants.py rename to db/constants.py diff --git a/collector_db/enums.py b/db/enums.py similarity index 100% rename from collector_db/enums.py rename to db/enums.py diff --git a/collector_db/helper_functions.py b/db/helper_functions.py similarity index 100% rename from collector_db/helper_functions.py rename to db/helper_functions.py diff --git a/collector_db/models.py b/db/models.py similarity index 99% rename from collector_db/models.py rename to db/models.py index 9134a0ad..83ca97b4 100644 --- a/collector_db/models.py +++ b/db/models.py @@ -6,7 +6,7 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.orm import declarative_base, relationship -from collector_db.enums import PGEnum, TaskType +from db.enums import PGEnum, TaskType from core.enums import BatchStatus, RecordType from util.helper_functions import get_enum_values diff --git a/docker-compose.yml b/docker-compose.yml index d813a97f..58cb29e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: build: context: . dockerfile: Dockerfile - command: pytest /app/tests/test_automated + command: pytest /app/tests/automated ports: - "8000:80" environment: diff --git a/html_tag_collector/DataClassTags.py b/html_tag_collector/DataClassTags.py index d39a0767..12a0ecc3 100644 --- a/html_tag_collector/DataClassTags.py +++ b/html_tag_collector/DataClassTags.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType @dataclass diff --git a/html_tag_collector/RootURLCache.py b/html_tag_collector/RootURLCache.py index 165be89d..b5f3f413 100644 --- a/html_tag_collector/RootURLCache.py +++ b/html_tag_collector/RootURLCache.py @@ -5,7 +5,7 @@ from aiohttp import ClientSession from bs4 import BeautifulSoup -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from html_tag_collector.constants import REQUEST_HEADERS DEBUG = False diff --git a/llm_api_logic/DeepSeekRecordClassifier.py b/llm_api_logic/DeepSeekRecordClassifier.py index 67f6fa09..e770f3c0 100644 --- a/llm_api_logic/DeepSeekRecordClassifier.py +++ b/llm_api_logic/DeepSeekRecordClassifier.py @@ -3,7 +3,7 @@ from openai import AsyncOpenAI -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo from core.enums import RecordType from llm_api_logic.LLMRecordClassifierBase import RecordClassifierBase diff --git a/llm_api_logic/LLMRecordClassifierBase.py b/llm_api_logic/LLMRecordClassifierBase.py index 85142aea..5648a90f 100644 --- a/llm_api_logic/LLMRecordClassifierBase.py +++ b/llm_api_logic/LLMRecordClassifierBase.py @@ -4,7 +4,7 @@ from openai import AsyncOpenAI -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo from llm_api_logic.RecordTypeStructuredOutput import RecordTypeStructuredOutput from llm_api_logic.constants import RECORD_CLASSIFICATION_QUERY_CONTENT from llm_api_logic.helpers import dictify_html_info diff --git a/llm_api_logic/helpers.py b/llm_api_logic/helpers.py index 3d5bde11..b8a81b13 100644 --- a/llm_api_logic/helpers.py +++ b/llm_api_logic/helpers.py @@ -1,4 +1,4 @@ -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo def dictify_html_info(html_infos: list[URLHTMLContentInfo]) -> dict[str, str]: diff --git a/tests/test_alembic/__init__.py b/tests/alembic/__init__.py similarity index 100% rename from tests/test_alembic/__init__.py rename to tests/alembic/__init__.py diff --git a/tests/test_alembic/conftest.py b/tests/alembic/conftest.py similarity index 95% rename from tests/test_alembic/conftest.py rename to tests/alembic/conftest.py index 8cd1d0ab..42a04a6b 100644 --- a/tests/test_alembic/conftest.py +++ b/tests/alembic/conftest.py @@ -3,7 +3,7 @@ from sqlalchemy import create_engine, inspect, MetaData from sqlalchemy.orm import scoped_session, sessionmaker -from collector_db.helper_functions import get_postgres_connection_string +from db.helper_functions import get_postgres_connection_string from tests.helpers.AlembicRunner import AlembicRunner diff --git a/tests/test_alembic/helpers.py b/tests/alembic/helpers.py similarity index 100% rename from tests/test_alembic/helpers.py rename to tests/alembic/helpers.py diff --git a/tests/test_alembic/test_revisions.py b/tests/alembic/test_revisions.py similarity index 98% rename from tests/test_alembic/test_revisions.py rename to tests/alembic/test_revisions.py index 9bc287d1..5ddd77e5 100644 --- a/tests/test_alembic/test_revisions.py +++ b/tests/alembic/test_revisions.py @@ -15,8 +15,8 @@ from sqlalchemy import text -from tests.test_alembic.helpers import columns_in_table -from tests.test_alembic.helpers import get_enum_values, table_creation_check +from tests.alembic.helpers import columns_in_table +from tests.alembic.helpers import get_enum_values, table_creation_check def test_base(alembic_runner): diff --git a/tests/test_automated/README.md b/tests/automated/README.md similarity index 100% rename from tests/test_automated/README.md rename to tests/automated/README.md diff --git a/tests/test_automated/__init__.py b/tests/automated/__init__.py similarity index 100% rename from tests/test_automated/__init__.py rename to tests/automated/__init__.py diff --git a/tests/test_automated/integration/__init__.py b/tests/automated/integration/__init__.py similarity index 100% rename from tests/test_automated/integration/__init__.py rename to tests/automated/integration/__init__.py diff --git a/tests/test_automated/integration/api/__init__.py b/tests/automated/integration/api/__init__.py similarity index 100% rename from tests/test_automated/integration/api/__init__.py rename to tests/automated/integration/api/__init__.py diff --git a/tests/test_automated/integration/api/conftest.py b/tests/automated/integration/api/conftest.py similarity index 97% rename from tests/test_automated/integration/api/conftest.py rename to tests/automated/integration/api/conftest.py index 2e7d8632..c709d202 100644 --- a/tests/test_automated/integration/api/conftest.py +++ b/tests/automated/integration/api/conftest.py @@ -15,7 +15,7 @@ from core.enums import BatchStatus from security_manager.SecurityManager import get_access_info, AccessInfo, Permissions, require_permission from tests.helpers.DBDataCreator import DBDataCreator -from tests.test_automated.integration.api.helpers.RequestValidator import RequestValidator +from tests.automated.integration.api.helpers.RequestValidator import RequestValidator @dataclass diff --git a/tests/test_automated/integration/api/helpers/RequestValidator.py b/tests/automated/integration/api/helpers/RequestValidator.py similarity index 98% rename from tests/test_automated/integration/api/helpers/RequestValidator.py rename to tests/automated/integration/api/helpers/RequestValidator.py index 3717da50..5fabd69b 100644 --- a/tests/test_automated/integration/api/helpers/RequestValidator.py +++ b/tests/automated/integration/api/helpers/RequestValidator.py @@ -5,10 +5,10 @@ from pydantic import BaseModel from starlette.testclient import TestClient -from collector_db.DTOs.BatchInfo import BatchInfo -from collector_db.DTOs.GetTaskStatusResponseInfo import GetTaskStatusResponseInfo -from collector_db.DTOs.TaskInfo import TaskInfo -from collector_db.enums import TaskType +from db.DTOs.BatchInfo import BatchInfo +from db.DTOs.GetTaskStatusResponseInfo import GetTaskStatusResponseInfo +from db.DTOs.TaskInfo import TaskInfo +from db.enums import TaskType from collector_manager.DTOs.ExampleInputDTO import ExampleInputDTO from collector_manager.enums import CollectorType from core.DTOs.AllAnnotationPostInfo import AllAnnotationPostInfo diff --git a/tests/test_automated/integration/api/helpers/__init__.py b/tests/automated/integration/api/helpers/__init__.py similarity index 100% rename from tests/test_automated/integration/api/helpers/__init__.py rename to tests/automated/integration/api/helpers/__init__.py diff --git a/tests/test_automated/integration/api/test_annotate.py b/tests/automated/integration/api/test_annotate.py similarity index 98% rename from tests/test_automated/integration/api/test_annotate.py rename to tests/automated/integration/api/test_annotate.py index 90181951..a3344b68 100644 --- a/tests/test_automated/integration/api/test_annotate.py +++ b/tests/automated/integration/api/test_annotate.py @@ -3,9 +3,9 @@ import pytest from fastapi import HTTPException -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo -from collector_db.DTOs.URLMapping import URLMapping -from collector_db.models import UserUrlAgencySuggestion, UserRelevantSuggestion, UserRecordTypeSuggestion +from db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.URLMapping import URLMapping +from db.models import UserUrlAgencySuggestion, UserRelevantSuggestion, UserRecordTypeSuggestion from core.DTOs.AllAnnotationPostInfo import AllAnnotationPostInfo from core.DTOs.GetNextRecordTypeAnnotationResponseInfo import GetNextRecordTypeAnnotationResponseOuterInfo from core.DTOs.GetNextRelevanceAnnotationResponseInfo import GetNextRelevanceAnnotationResponseOuterInfo @@ -19,7 +19,7 @@ setup_for_get_next_url_for_final_review from html_tag_collector.DataClassTags import ResponseHTMLInfo from tests.helpers.DBDataCreator import BatchURLCreationInfo -from tests.test_automated.integration.api.conftest import MOCK_USER_ID +from tests.automated.integration.api.conftest import MOCK_USER_ID def check_url_mappings_match( map_1: URLMapping, diff --git a/tests/test_automated/integration/api/test_batch.py b/tests/automated/integration/api/test_batch.py similarity index 97% rename from tests/test_automated/integration/api/test_batch.py rename to tests/automated/integration/api/test_batch.py index 961b1a30..2f7e2ebb 100644 --- a/tests/test_automated/integration/api/test_batch.py +++ b/tests/automated/integration/api/test_batch.py @@ -3,8 +3,8 @@ import pytest -from collector_db.DTOs.BatchInfo import BatchInfo -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.BatchInfo import BatchInfo +from db.DTOs.InsertURLsInfo import InsertURLsInfo from collector_manager.DTOs.ExampleInputDTO import ExampleInputDTO from collector_manager.enums import CollectorType, URLStatus from core.enums import BatchStatus diff --git a/tests/test_automated/integration/api/test_duplicates.py b/tests/automated/integration/api/test_duplicates.py similarity index 92% rename from tests/test_automated/integration/api/test_duplicates.py rename to tests/automated/integration/api/test_duplicates.py index 6c6c42ce..654a9c65 100644 --- a/tests/test_automated/integration/api/test_duplicates.py +++ b/tests/automated/integration/api/test_duplicates.py @@ -3,9 +3,9 @@ import pytest -from collector_db.DTOs.BatchInfo import BatchInfo +from db.DTOs.BatchInfo import BatchInfo from collector_manager.DTOs.ExampleInputDTO import ExampleInputDTO -from tests.test_automated.integration.api.conftest import disable_task_trigger +from tests.automated.integration.api.conftest import disable_task_trigger @pytest.mark.asyncio diff --git a/tests/test_automated/integration/api/test_example_collector.py b/tests/automated/integration/api/test_example_collector.py similarity index 95% rename from tests/test_automated/integration/api/test_example_collector.py rename to tests/automated/integration/api/test_example_collector.py index 0b3cf30f..83d1ad6d 100644 --- a/tests/test_automated/integration/api/test_example_collector.py +++ b/tests/automated/integration/api/test_example_collector.py @@ -3,8 +3,8 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.BatchInfo import BatchInfo +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.BatchInfo import BatchInfo from collector_manager.DTOs.ExampleInputDTO import ExampleInputDTO from collector_manager.ExampleCollector import ExampleCollector from collector_manager.enums import CollectorType @@ -14,7 +14,7 @@ from core.DTOs.GetBatchStatusResponse import GetBatchStatusResponse from core.enums import BatchStatus from tests.helpers.patch_functions import block_sleep -from tests.test_automated.integration.api.conftest import disable_task_trigger +from tests.automated.integration.api.conftest import disable_task_trigger @pytest.mark.asyncio diff --git a/tests/test_automated/integration/api/test_manual_batch.py b/tests/automated/integration/api/test_manual_batch.py similarity index 98% rename from tests/test_automated/integration/api/test_manual_batch.py rename to tests/automated/integration/api/test_manual_batch.py index e9a101eb..1c0a2ecc 100644 --- a/tests/test_automated/integration/api/test_manual_batch.py +++ b/tests/automated/integration/api/test_manual_batch.py @@ -1,7 +1,7 @@ import pytest -from collector_db.models import Batch, URL, URLOptionalDataSourceMetadata +from db.models import Batch, URL, URLOptionalDataSourceMetadata from collector_manager.enums import CollectorType from core.DTOs.ManualBatchInputDTO import ManualBatchInnerInputDTO, ManualBatchInputDTO from core.enums import RecordType diff --git a/tests/test_automated/integration/api/test_metrics.py b/tests/automated/integration/api/test_metrics.py similarity index 100% rename from tests/test_automated/integration/api/test_metrics.py rename to tests/automated/integration/api/test_metrics.py diff --git a/tests/test_automated/integration/api/test_review.py b/tests/automated/integration/api/test_review.py similarity index 97% rename from tests/test_automated/integration/api/test_review.py rename to tests/automated/integration/api/test_review.py index facd6926..a034b740 100644 --- a/tests/test_automated/integration/api/test_review.py +++ b/tests/automated/integration/api/test_review.py @@ -1,7 +1,7 @@ import pytest -from collector_db.constants import PLACEHOLDER_AGENCY_NAME -from collector_db.models import URL, URLOptionalDataSourceMetadata, ConfirmedURLAgency, Agency +from db.constants import PLACEHOLDER_AGENCY_NAME +from db.models import URL, URLOptionalDataSourceMetadata, ConfirmedURLAgency, Agency from collector_manager.enums import URLStatus from core.DTOs.FinalReviewApprovalInfo import FinalReviewApprovalInfo, FinalReviewBaseInfo, RejectionReason, \ FinalReviewRejectionInfo diff --git a/tests/test_automated/integration/api/test_root.py b/tests/automated/integration/api/test_root.py similarity index 100% rename from tests/test_automated/integration/api/test_root.py rename to tests/automated/integration/api/test_root.py diff --git a/tests/test_automated/integration/api/test_search.py b/tests/automated/integration/api/test_search.py similarity index 100% rename from tests/test_automated/integration/api/test_search.py rename to tests/automated/integration/api/test_search.py diff --git a/tests/test_automated/integration/api/test_task.py b/tests/automated/integration/api/test_task.py similarity index 93% rename from tests/test_automated/integration/api/test_task.py rename to tests/automated/integration/api/test_task.py index 547b0eb8..c13f97f9 100644 --- a/tests/test_automated/integration/api/test_task.py +++ b/tests/automated/integration/api/test_task.py @@ -1,7 +1,7 @@ import pytest -from collector_db.enums import TaskType -from tests.test_automated.integration.api.conftest import APITestHelper +from db.enums import TaskType +from tests.automated.integration.api.conftest import APITestHelper async def task_setup(ath: APITestHelper) -> int: diff --git a/tests/test_automated/integration/api/test_url.py b/tests/automated/integration/api/test_url.py similarity index 95% rename from tests/test_automated/integration/api/test_url.py rename to tests/automated/integration/api/test_url.py index fccd8e4e..9068af5e 100644 --- a/tests/test_automated/integration/api/test_url.py +++ b/tests/automated/integration/api/test_url.py @@ -1,6 +1,6 @@ import pytest -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.InsertURLsInfo import InsertURLsInfo from core.DTOs.GetURLsResponseInfo import GetURLsResponseInfo diff --git a/tests/test_automated/integration/collector_db/README.md b/tests/automated/integration/collector_db/README.md similarity index 100% rename from tests/test_automated/integration/collector_db/README.md rename to tests/automated/integration/collector_db/README.md diff --git a/tests/test_automated/integration/collector_db/__init__.py b/tests/automated/integration/collector_db/__init__.py similarity index 100% rename from tests/test_automated/integration/collector_db/__init__.py rename to tests/automated/integration/collector_db/__init__.py diff --git a/tests/test_automated/integration/collector_db/test_database_structure.py b/tests/automated/integration/collector_db/test_database_structure.py similarity index 97% rename from tests/test_automated/integration/collector_db/test_database_structure.py rename to tests/automated/integration/collector_db/test_database_structure.py index 6d82631c..88b186ad 100644 --- a/tests/test_automated/integration/collector_db/test_database_structure.py +++ b/tests/automated/integration/collector_db/test_database_structure.py @@ -16,10 +16,10 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.exc import DataError, DBAPIError -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo -from collector_db.enums import URLHTMLContentType -from collector_db.helper_functions import get_postgres_connection_string -from collector_db.models import Base, Agency +from db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.enums import URLHTMLContentType +from db.helper_functions import get_postgres_connection_string +from db.models import Base, Agency from collector_manager.enums import CollectorType, URLStatus from core.DTOs.URLAgencySuggestionInfo import URLAgencySuggestionInfo from core.enums import BatchStatus, SuggestionType diff --git a/tests/test_automated/integration/collector_db/test_db_client.py b/tests/automated/integration/collector_db/test_db_client.py similarity index 98% rename from tests/test_automated/integration/collector_db/test_db_client.py rename to tests/automated/integration/collector_db/test_db_client.py index b0c5e91d..644bd500 100644 --- a/tests/test_automated/integration/collector_db/test_db_client.py +++ b/tests/automated/integration/collector_db/test_db_client.py @@ -3,14 +3,14 @@ import pytest from fastapi import HTTPException -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.BatchInfo import BatchInfo -from collector_db.DTOs.LogInfo import LogInfo -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.DTOs.URLInfo import URLInfo -from collector_db.DTOs.URLMapping import URLMapping -from collector_db.constants import PLACEHOLDER_AGENCY_NAME -from collector_db.models import URL, ReviewingUserURL, URLOptionalDataSourceMetadata, ConfirmedURLAgency, Agency +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.BatchInfo import BatchInfo +from db.DTOs.LogInfo import LogInfo +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.DTOs.URLInfo import URLInfo +from db.DTOs.URLMapping import URLMapping +from db.constants import PLACEHOLDER_AGENCY_NAME +from db.models import URL, ReviewingUserURL, URLOptionalDataSourceMetadata, ConfirmedURLAgency, Agency from collector_manager.enums import URLStatus from core.DTOs.FinalReviewApprovalInfo import FinalReviewApprovalInfo from core.enums import BatchStatus, RecordType, SuggestionType, SuggestedStatus diff --git a/tests/test_automated/integration/conftest.py b/tests/automated/integration/conftest.py similarity index 93% rename from tests/test_automated/integration/conftest.py rename to tests/automated/integration/conftest.py index 70c79c22..3912f3e8 100644 --- a/tests/test_automated/integration/conftest.py +++ b/tests/automated/integration/conftest.py @@ -2,7 +2,7 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from collector_manager.AsyncCollectorManager import AsyncCollectorManager from core.AsyncCore import AsyncCore from core.AsyncCoreLogger import AsyncCoreLogger diff --git a/tests/test_automated/integration/core/README.md b/tests/automated/integration/core/README.md similarity index 100% rename from tests/test_automated/integration/core/README.md rename to tests/automated/integration/core/README.md diff --git a/tests/test_automated/integration/core/__init__.py b/tests/automated/integration/core/__init__.py similarity index 100% rename from tests/test_automated/integration/core/__init__.py rename to tests/automated/integration/core/__init__.py diff --git a/tests/test_automated/integration/core/helpers/README.md b/tests/automated/integration/core/helpers/README.md similarity index 100% rename from tests/test_automated/integration/core/helpers/README.md rename to tests/automated/integration/core/helpers/README.md diff --git a/tests/test_automated/integration/core/helpers/__init__.py b/tests/automated/integration/core/helpers/__init__.py similarity index 100% rename from tests/test_automated/integration/core/helpers/__init__.py rename to tests/automated/integration/core/helpers/__init__.py diff --git a/tests/test_automated/integration/core/helpers/constants.py b/tests/automated/integration/core/helpers/constants.py similarity index 100% rename from tests/test_automated/integration/core/helpers/constants.py rename to tests/automated/integration/core/helpers/constants.py diff --git a/tests/test_automated/integration/core/test_async_core.py b/tests/automated/integration/core/test_async_core.py similarity index 97% rename from tests/test_automated/integration/core/test_async_core.py rename to tests/automated/integration/core/test_async_core.py index e43b519f..7fa3d757 100644 --- a/tests/test_automated/integration/core/test_async_core.py +++ b/tests/automated/integration/core/test_async_core.py @@ -3,9 +3,9 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.enums import TaskType -from collector_db.models import Task +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.enums import TaskType +from db.models import Task from core.AsyncCore import AsyncCore from core.DTOs.TaskOperatorRunInfo import TaskOperatorRunInfo, TaskOperatorOutcome from core.TaskManager import TaskManager diff --git a/tests/test_automated/integration/core/test_example_collector_lifecycle.py b/tests/automated/integration/core/test_example_collector_lifecycle.py similarity index 97% rename from tests/test_automated/integration/core/test_example_collector_lifecycle.py rename to tests/automated/integration/core/test_example_collector_lifecycle.py index 18411457..f094e5b7 100644 --- a/tests/test_automated/integration/core/test_example_collector_lifecycle.py +++ b/tests/automated/integration/core/test_example_collector_lifecycle.py @@ -2,7 +2,7 @@ import pytest -from collector_db.DTOs.BatchInfo import BatchInfo +from db.DTOs.BatchInfo import BatchInfo from collector_manager.DTOs.ExampleInputDTO import ExampleInputDTO from collector_manager.enums import CollectorType, URLStatus from core.AsyncCore import AsyncCore diff --git a/tests/test_automated/integration/html_tag_collector/__init__.py b/tests/automated/integration/html_tag_collector/__init__.py similarity index 100% rename from tests/test_automated/integration/html_tag_collector/__init__.py rename to tests/automated/integration/html_tag_collector/__init__.py diff --git a/tests/test_automated/integration/html_tag_collector/test_root_url_cache.py b/tests/automated/integration/html_tag_collector/test_root_url_cache.py similarity index 100% rename from tests/test_automated/integration/html_tag_collector/test_root_url_cache.py rename to tests/automated/integration/html_tag_collector/test_root_url_cache.py diff --git a/tests/test_automated/integration/security_manager/__init__.py b/tests/automated/integration/security_manager/__init__.py similarity index 100% rename from tests/test_automated/integration/security_manager/__init__.py rename to tests/automated/integration/security_manager/__init__.py diff --git a/tests/test_automated/integration/security_manager/test_security_manager.py b/tests/automated/integration/security_manager/test_security_manager.py similarity index 100% rename from tests/test_automated/integration/security_manager/test_security_manager.py rename to tests/automated/integration/security_manager/test_security_manager.py diff --git a/tests/test_automated/integration/tasks/__init__.py b/tests/automated/integration/tasks/__init__.py similarity index 100% rename from tests/test_automated/integration/tasks/__init__.py rename to tests/automated/integration/tasks/__init__.py diff --git a/tests/test_automated/integration/tasks/conftest.py b/tests/automated/integration/tasks/conftest.py similarity index 100% rename from tests/test_automated/integration/tasks/conftest.py rename to tests/automated/integration/tasks/conftest.py diff --git a/tests/test_automated/integration/tasks/test_agency_preannotation_task.py b/tests/automated/integration/tasks/test_agency_preannotation_task.py similarity index 99% rename from tests/test_automated/integration/tasks/test_agency_preannotation_task.py rename to tests/automated/integration/tasks/test_agency_preannotation_task.py index e6278292..c0de5c52 100644 --- a/tests/test_automated/integration/tasks/test_agency_preannotation_task.py +++ b/tests/automated/integration/tasks/test_agency_preannotation_task.py @@ -7,7 +7,7 @@ from tests.helpers.test_batch_creation_parameters import TestBatchCreationParameters, TestURLCreationParameters from source_collectors.muckrock.MuckrockAPIInterface import MuckrockAPIInterface, AgencyLookupResponseType, AgencyLookupResponse -from collector_db.models import Agency, AutomatedUrlAgencySuggestion +from db.models import Agency, AutomatedUrlAgencySuggestion from collector_manager.enums import CollectorType, URLStatus from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome from core.DTOs.URLAgencySuggestionInfo import URLAgencySuggestionInfo diff --git a/tests/test_automated/integration/tasks/test_example_task.py b/tests/automated/integration/tasks/test_example_task.py similarity index 97% rename from tests/test_automated/integration/tasks/test_example_task.py rename to tests/automated/integration/tasks/test_example_task.py index 2211458c..c0515103 100644 --- a/tests/test_automated/integration/tasks/test_example_task.py +++ b/tests/automated/integration/tasks/test_example_task.py @@ -2,7 +2,7 @@ import pytest -from collector_db.enums import TaskType +from db.enums import TaskType from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome from core.classes.task_operators.TaskOperatorBase import TaskOperatorBase from tests.helpers.DBDataCreator import DBDataCreator diff --git a/tests/test_automated/integration/tasks/test_submit_approved_url_task.py b/tests/automated/integration/tasks/test_submit_approved_url_task.py similarity index 98% rename from tests/test_automated/integration/tasks/test_submit_approved_url_task.py rename to tests/automated/integration/tasks/test_submit_approved_url_task.py index 1477915f..d5453005 100644 --- a/tests/test_automated/integration/tasks/test_submit_approved_url_task.py +++ b/tests/automated/integration/tasks/test_submit_approved_url_task.py @@ -4,8 +4,8 @@ import pytest from deepdiff import DeepDiff -from collector_db.enums import TaskType -from collector_db.models import URL, URLErrorInfo, URLDataSource +from db.enums import TaskType +from db.models import URL, URLErrorInfo, URLDataSource from collector_manager.enums import URLStatus from core.DTOs.FinalReviewApprovalInfo import FinalReviewApprovalInfo from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome diff --git a/tests/test_automated/integration/tasks/test_url_404_probe.py b/tests/automated/integration/tasks/test_url_404_probe.py similarity index 99% rename from tests/test_automated/integration/tasks/test_url_404_probe.py rename to tests/automated/integration/tasks/test_url_404_probe.py index e248363a..a897a59e 100644 --- a/tests/test_automated/integration/tasks/test_url_404_probe.py +++ b/tests/automated/integration/tasks/test_url_404_probe.py @@ -5,7 +5,7 @@ import pytest from aiohttp import ClientResponseError, RequestInfo -from collector_db.models import URLProbedFor404, URL +from db.models import URLProbedFor404, URL from collector_manager.enums import URLStatus from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome from core.classes.task_operators.URL404ProbeTaskOperator import URL404ProbeTaskOperator diff --git a/tests/test_automated/integration/tasks/test_url_duplicate_task.py b/tests/automated/integration/tasks/test_url_duplicate_task.py similarity index 96% rename from tests/test_automated/integration/tasks/test_url_duplicate_task.py rename to tests/automated/integration/tasks/test_url_duplicate_task.py index d66cfe27..0987a2f4 100644 --- a/tests/test_automated/integration/tasks/test_url_duplicate_task.py +++ b/tests/automated/integration/tasks/test_url_duplicate_task.py @@ -3,8 +3,8 @@ import pytest -from collector_db.DTOs.URLMapping import URLMapping -from collector_db.models import URL, URLCheckedForDuplicate +from db.DTOs.URLMapping import URLMapping +from db.models import URL, URLCheckedForDuplicate from collector_manager.enums import URLStatus from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome from core.classes.task_operators.URLDuplicateTaskOperator import URLDuplicateTaskOperator diff --git a/tests/test_automated/integration/tasks/test_url_html_task.py b/tests/automated/integration/tasks/test_url_html_task.py similarity index 97% rename from tests/test_automated/integration/tasks/test_url_html_task.py rename to tests/automated/integration/tasks/test_url_html_task.py index aeb0db7f..7926b26b 100644 --- a/tests/test_automated/integration/tasks/test_url_html_task.py +++ b/tests/automated/integration/tasks/test_url_html_task.py @@ -5,8 +5,8 @@ import pytest from aiohttp import ClientError, ClientResponseError, RequestInfo -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.enums import TaskType from collector_manager.enums import URLStatus from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome from core.classes.task_operators.URLHTMLTaskOperator import URLHTMLTaskOperator diff --git a/tests/test_automated/integration/tasks/test_url_miscellaneous_metadata_task.py b/tests/automated/integration/tasks/test_url_miscellaneous_metadata_task.py similarity index 98% rename from tests/test_automated/integration/tasks/test_url_miscellaneous_metadata_task.py rename to tests/automated/integration/tasks/test_url_miscellaneous_metadata_task.py index 526efa70..2b63c33c 100644 --- a/tests/test_automated/integration/tasks/test_url_miscellaneous_metadata_task.py +++ b/tests/automated/integration/tasks/test_url_miscellaneous_metadata_task.py @@ -2,7 +2,7 @@ import pytest -from collector_db.models import URL, URLOptionalDataSourceMetadata +from db.models import URL, URLOptionalDataSourceMetadata from collector_manager.enums import CollectorType from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome from core.classes.task_operators.URLMiscellaneousMetadataTaskOperator import URLMiscellaneousMetadataTaskOperator diff --git a/tests/test_automated/integration/tasks/test_url_record_type_task.py b/tests/automated/integration/tasks/test_url_record_type_task.py similarity index 95% rename from tests/test_automated/integration/tasks/test_url_record_type_task.py rename to tests/automated/integration/tasks/test_url_record_type_task.py index c941bcf7..1f26812a 100644 --- a/tests/test_automated/integration/tasks/test_url_record_type_task.py +++ b/tests/automated/integration/tasks/test_url_record_type_task.py @@ -2,8 +2,8 @@ import pytest -from collector_db.enums import TaskType -from collector_db.models import AutoRecordTypeSuggestion +from db.enums import TaskType +from db.models import AutoRecordTypeSuggestion from core.DTOs.TaskOperatorRunInfo import TaskOperatorOutcome from core.classes.task_operators.URLRecordTypeTaskOperator import URLRecordTypeTaskOperator from core.enums import RecordType diff --git a/tests/test_automated/unit/__init__.py b/tests/automated/unit/__init__.py similarity index 100% rename from tests/test_automated/unit/__init__.py rename to tests/automated/unit/__init__.py diff --git a/tests/test_automated/unit/core/__init__.py b/tests/automated/unit/core/__init__.py similarity index 100% rename from tests/test_automated/unit/core/__init__.py rename to tests/automated/unit/core/__init__.py diff --git a/tests/test_automated/unit/core/test_core_logger.py b/tests/automated/unit/core/test_core_logger.py similarity index 94% rename from tests/test_automated/unit/core/test_core_logger.py rename to tests/automated/unit/core/test_core_logger.py index b0d52055..f60f989c 100644 --- a/tests/test_automated/unit/core/test_core_logger.py +++ b/tests/automated/unit/core/test_core_logger.py @@ -3,7 +3,7 @@ import pytest -from collector_db.DTOs.LogInfo import LogInfo +from db.DTOs.LogInfo import LogInfo from core.AsyncCoreLogger import AsyncCoreLogger diff --git a/tests/test_automated/unit/dto/__init__.py b/tests/automated/unit/dto/__init__.py similarity index 100% rename from tests/test_automated/unit/dto/__init__.py rename to tests/automated/unit/dto/__init__.py diff --git a/tests/test_automated/unit/dto/test_all_annotation_post_info.py b/tests/automated/unit/dto/test_all_annotation_post_info.py similarity index 100% rename from tests/test_automated/unit/dto/test_all_annotation_post_info.py rename to tests/automated/unit/dto/test_all_annotation_post_info.py diff --git a/tests/test_automated/unit/security_manager/__init__.py b/tests/automated/unit/security_manager/__init__.py similarity index 100% rename from tests/test_automated/unit/security_manager/__init__.py rename to tests/automated/unit/security_manager/__init__.py diff --git a/tests/test_automated/unit/security_manager/test_security_manager.py b/tests/automated/unit/security_manager/test_security_manager.py similarity index 100% rename from tests/test_automated/unit/security_manager/test_security_manager.py rename to tests/automated/unit/security_manager/test_security_manager.py diff --git a/tests/test_automated/unit/source_collectors/__init__.py b/tests/automated/unit/source_collectors/__init__.py similarity index 100% rename from tests/test_automated/unit/source_collectors/__init__.py rename to tests/automated/unit/source_collectors/__init__.py diff --git a/tests/test_automated/unit/source_collectors/test_autogoogler_collector.py b/tests/automated/unit/source_collectors/test_autogoogler_collector.py similarity index 92% rename from tests/test_automated/unit/source_collectors/test_autogoogler_collector.py rename to tests/automated/unit/source_collectors/test_autogoogler_collector.py index c3fafa61..dc5de285 100644 --- a/tests/test_automated/unit/source_collectors/test_autogoogler_collector.py +++ b/tests/automated/unit/source_collectors/test_autogoogler_collector.py @@ -2,8 +2,8 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLInfo import URLInfo +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLInfo import URLInfo from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.auto_googler.AutoGooglerCollector import AutoGooglerCollector from source_collectors.auto_googler.DTOs import GoogleSearchQueryResultsInnerDTO, AutoGooglerInputDTO diff --git a/tests/test_automated/unit/source_collectors/test_ckan_collector.py b/tests/automated/unit/source_collectors/test_ckan_collector.py similarity index 97% rename from tests/test_automated/unit/source_collectors/test_ckan_collector.py rename to tests/automated/unit/source_collectors/test_ckan_collector.py index e0e9ee47..747b0852 100644 --- a/tests/test_automated/unit/source_collectors/test_ckan_collector.py +++ b/tests/automated/unit/source_collectors/test_ckan_collector.py @@ -4,7 +4,7 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.ckan.CKANCollector import CKANCollector from source_collectors.ckan.DTOs import CKANInputDTO diff --git a/tests/test_automated/unit/source_collectors/test_common_crawl_collector.py b/tests/automated/unit/source_collectors/test_common_crawl_collector.py similarity index 93% rename from tests/test_automated/unit/source_collectors/test_common_crawl_collector.py rename to tests/automated/unit/source_collectors/test_common_crawl_collector.py index 1c5aa6ee..6023b8cf 100644 --- a/tests/test_automated/unit/source_collectors/test_common_crawl_collector.py +++ b/tests/automated/unit/source_collectors/test_common_crawl_collector.py @@ -2,8 +2,8 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLInfo import URLInfo +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLInfo import URLInfo from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.common_crawler.CommonCrawlerCollector import CommonCrawlerCollector from source_collectors.common_crawler.DTOs import CommonCrawlerInputDTO diff --git a/tests/test_automated/unit/source_collectors/test_example_collector.py b/tests/automated/unit/source_collectors/test_example_collector.py similarity index 90% rename from tests/test_automated/unit/source_collectors/test_example_collector.py rename to tests/automated/unit/source_collectors/test_example_collector.py index b770d952..e5d113cc 100644 --- a/tests/test_automated/unit/source_collectors/test_example_collector.py +++ b/tests/automated/unit/source_collectors/test_example_collector.py @@ -1,6 +1,6 @@ from unittest.mock import AsyncMock -from collector_db.DatabaseClient import DatabaseClient +from db.DatabaseClient import DatabaseClient from collector_manager.DTOs.ExampleInputDTO import ExampleInputDTO from collector_manager.ExampleCollector import ExampleCollector from core.AsyncCoreLogger import AsyncCoreLogger diff --git a/tests/test_automated/unit/source_collectors/test_muckrock_collectors.py b/tests/automated/unit/source_collectors/test_muckrock_collectors.py similarity index 98% rename from tests/test_automated/unit/source_collectors/test_muckrock_collectors.py rename to tests/automated/unit/source_collectors/test_muckrock_collectors.py index 100fbb6e..cd49ffb6 100644 --- a/tests/test_automated/unit/source_collectors/test_muckrock_collectors.py +++ b/tests/automated/unit/source_collectors/test_muckrock_collectors.py @@ -3,8 +3,8 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLInfo import URLInfo +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLInfo import URLInfo from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.muckrock.DTOs import MuckrockSimpleSearchCollectorInputDTO, \ MuckrockCountySearchCollectorInputDTO, MuckrockAllFOIARequestsCollectorInputDTO diff --git a/tests/test_automated/unit/test_function_trigger.py b/tests/automated/unit/test_function_trigger.py similarity index 100% rename from tests/test_automated/unit/test_function_trigger.py rename to tests/automated/unit/test_function_trigger.py diff --git a/tests/conftest.py b/tests/conftest.py index c8f4bd64..99281103 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,10 +3,10 @@ from sqlalchemy import create_engine, inspect, MetaData from sqlalchemy.orm import scoped_session, sessionmaker -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DatabaseClient import DatabaseClient -from collector_db.helper_functions import get_postgres_connection_string -from collector_db.models import Base +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DatabaseClient import DatabaseClient +from db.helper_functions import get_postgres_connection_string +from db.models import Base from core.EnvVarManager import EnvVarManager from tests.helpers.AlembicRunner import AlembicRunner from tests.helpers.DBDataCreator import DBDataCreator diff --git a/tests/helpers/DBDataCreator.py b/tests/helpers/DBDataCreator.py index 2ccf47fa..a0036e2a 100644 --- a/tests/helpers/DBDataCreator.py +++ b/tests/helpers/DBDataCreator.py @@ -5,16 +5,16 @@ from pydantic import BaseModel, model_validator -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.BatchInfo import BatchInfo -from collector_db.DTOs.DuplicateInfo import DuplicateInsertInfo -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo -from collector_db.DTOs.URLErrorInfos import URLErrorPydanticInfo -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType -from collector_db.DTOs.URLInfo import URLInfo -from collector_db.DTOs.URLMapping import URLMapping -from collector_db.DatabaseClient import DatabaseClient -from collector_db.enums import TaskType +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.BatchInfo import BatchInfo +from db.DTOs.DuplicateInfo import DuplicateInsertInfo +from db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.URLErrorInfos import URLErrorPydanticInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo, HTMLContentType +from db.DTOs.URLInfo import URLInfo +from db.DTOs.URLMapping import URLMapping +from db.DatabaseClient import DatabaseClient +from db.enums import TaskType from collector_manager.enums import CollectorType, URLStatus from core.DTOs.FinalReviewApprovalInfo import FinalReviewApprovalInfo, RejectionReason from core.DTOs.URLAgencySuggestionInfo import URLAgencySuggestionInfo diff --git a/tests/helpers/assert_functions.py b/tests/helpers/assert_functions.py index ef379d3e..7deaacc3 100644 --- a/tests/helpers/assert_functions.py +++ b/tests/helpers/assert_functions.py @@ -1,5 +1,5 @@ -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.models import Task +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.models import Task async def assert_database_has_no_tasks(adb_client: AsyncDatabaseClient): diff --git a/tests/helpers/complex_test_data_functions.py b/tests/helpers/complex_test_data_functions.py index bc03020f..32dfca02 100644 --- a/tests/helpers/complex_test_data_functions.py +++ b/tests/helpers/complex_test_data_functions.py @@ -2,8 +2,8 @@ from pydantic import BaseModel -from collector_db.DTOs.InsertURLsInfo import InsertURLsInfo -from collector_db.DTOs.URLMapping import URLMapping +from db.DTOs.InsertURLsInfo import InsertURLsInfo +from db.DTOs.URLMapping import URLMapping from collector_manager.enums import URLStatus from core.enums import RecordType, SuggestionType from tests.helpers.DBDataCreator import BatchURLCreationInfo diff --git a/tests/manual/core/lifecycle/test_auto_googler_lifecycle.py b/tests/manual/core/lifecycle/test_auto_googler_lifecycle.py index 9e5c0e49..d832c2a8 100644 --- a/tests/manual/core/lifecycle/test_auto_googler_lifecycle.py +++ b/tests/manual/core/lifecycle/test_auto_googler_lifecycle.py @@ -3,7 +3,7 @@ import dotenv import api.dependencies -from collector_db.DTOs.BatchInfo import BatchInfo +from db.DTOs.BatchInfo import BatchInfo from collector_manager.enums import CollectorType from core.enums import BatchStatus from test_automated.integration.core.helpers.common_test_procedures import run_collector_and_wait_for_completion diff --git a/tests/manual/core/lifecycle/test_ckan_lifecycle.py b/tests/manual/core/lifecycle/test_ckan_lifecycle.py index 4e87bbbd..2d4a4f7a 100644 --- a/tests/manual/core/lifecycle/test_ckan_lifecycle.py +++ b/tests/manual/core/lifecycle/test_ckan_lifecycle.py @@ -1,6 +1,6 @@ import api.dependencies -from collector_db.DTOs.BatchInfo import BatchInfo +from db.DTOs.BatchInfo import BatchInfo from collector_manager.enums import CollectorType from core.enums import BatchStatus from source_collectors.ckan.search_terms import group_search, package_search, organization_search diff --git a/tests/manual/core/lifecycle/test_muckrock_lifecycles.py b/tests/manual/core/lifecycle/test_muckrock_lifecycles.py index 72d2d9fc..26e4aa36 100644 --- a/tests/manual/core/lifecycle/test_muckrock_lifecycles.py +++ b/tests/manual/core/lifecycle/test_muckrock_lifecycles.py @@ -1,6 +1,6 @@ import api.dependencies -from collector_db.DTOs.BatchInfo import BatchInfo +from db.DTOs.BatchInfo import BatchInfo from collector_manager.enums import CollectorType from core.enums import BatchStatus from test_automated.integration.core.helpers.common_test_procedures import run_collector_and_wait_for_completion diff --git a/tests/manual/html_collector/test_html_tag_collector_integration.py b/tests/manual/html_collector/test_html_tag_collector_integration.py index 228f4b68..5777f907 100644 --- a/tests/manual/html_collector/test_html_tag_collector_integration.py +++ b/tests/manual/html_collector/test_html_tag_collector_integration.py @@ -1,7 +1,7 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient -from collector_db.DTOs.URLInfo import URLInfo +from db.AsyncDatabaseClient import AsyncDatabaseClient +from db.DTOs.URLInfo import URLInfo from core.classes.task_operators.URLHTMLTaskOperator import URLHTMLTaskOperator from tests.helpers.DBDataCreator import DBDataCreator from html_tag_collector.ResponseParser import HTMLResponseParser diff --git a/tests/manual/llm_api_logic/test_deepseek_record_classifier.py b/tests/manual/llm_api_logic/test_deepseek_record_classifier.py index b0a6c1fb..18363a71 100644 --- a/tests/manual/llm_api_logic/test_deepseek_record_classifier.py +++ b/tests/manual/llm_api_logic/test_deepseek_record_classifier.py @@ -1,12 +1,12 @@ import pytest -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo from llm_api_logic.DeepSeekRecordClassifier import DeepSeekRecordClassifier @pytest.mark.asyncio async def test_deepseek_record_classifier(): - from collector_db.DTOs.URLHTMLContentInfo import HTMLContentType as hct + from db.DTOs.URLHTMLContentInfo import HTMLContentType as hct d = { hct.TITLE: "Oath of Office for Newly Promoted Corporal Lumpkin with Acworth Police – City of Acworth, GA", diff --git a/tests/manual/llm_api_logic/test_openai_record_classifier.py b/tests/manual/llm_api_logic/test_openai_record_classifier.py index 72d474d2..57b56a54 100644 --- a/tests/manual/llm_api_logic/test_openai_record_classifier.py +++ b/tests/manual/llm_api_logic/test_openai_record_classifier.py @@ -1,12 +1,12 @@ import pytest -from collector_db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo +from db.DTOs.URLHTMLContentInfo import URLHTMLContentInfo from llm_api_logic.OpenAIRecordClassifier import OpenAIRecordClassifier @pytest.mark.asyncio async def test_openai_record_classifier(): - from collector_db.DTOs.URLHTMLContentInfo import HTMLContentType as hct + from db.DTOs.URLHTMLContentInfo import HTMLContentType as hct d = { hct.TITLE: "Oath of Office for Newly Promoted Corporal Lumpkin with Acworth Police – City of Acworth, GA", diff --git a/tests/manual/source_collectors/test_autogoogler_collector.py b/tests/manual/source_collectors/test_autogoogler_collector.py index c9942106..cabdcc1e 100644 --- a/tests/manual/source_collectors/test_autogoogler_collector.py +++ b/tests/manual/source_collectors/test_autogoogler_collector.py @@ -2,7 +2,7 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.auto_googler.AutoGooglerCollector import AutoGooglerCollector from source_collectors.auto_googler.DTOs import AutoGooglerInputDTO diff --git a/tests/manual/source_collectors/test_ckan_collector.py b/tests/manual/source_collectors/test_ckan_collector.py index f9deaf02..e6a6c1f8 100644 --- a/tests/manual/source_collectors/test_ckan_collector.py +++ b/tests/manual/source_collectors/test_ckan_collector.py @@ -3,7 +3,7 @@ import pytest from marshmallow import Schema, fields -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.ckan.CKANCollector import CKANCollector from source_collectors.ckan.DTOs import CKANInputDTO diff --git a/tests/manual/source_collectors/test_common_crawler_collector.py b/tests/manual/source_collectors/test_common_crawler_collector.py index cb1c4f78..12be9ec7 100644 --- a/tests/manual/source_collectors/test_common_crawler_collector.py +++ b/tests/manual/source_collectors/test_common_crawler_collector.py @@ -3,7 +3,7 @@ import pytest from marshmallow import Schema, fields -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.common_crawler.CommonCrawlerCollector import CommonCrawlerCollector from source_collectors.common_crawler.DTOs import CommonCrawlerInputDTO diff --git a/tests/manual/source_collectors/test_muckrock_collectors.py b/tests/manual/source_collectors/test_muckrock_collectors.py index 49bfa5fb..c30473df 100644 --- a/tests/manual/source_collectors/test_muckrock_collectors.py +++ b/tests/manual/source_collectors/test_muckrock_collectors.py @@ -2,14 +2,14 @@ import pytest -from collector_db.AsyncDatabaseClient import AsyncDatabaseClient +from db.AsyncDatabaseClient import AsyncDatabaseClient from core.AsyncCoreLogger import AsyncCoreLogger from source_collectors.muckrock.DTOs import MuckrockSimpleSearchCollectorInputDTO, \ MuckrockCountySearchCollectorInputDTO, MuckrockAllFOIARequestsCollectorInputDTO from source_collectors.muckrock.classes.MuckrockCollector import MuckrockSimpleSearchCollector, \ MuckrockCountyLevelSearchCollector, MuckrockAllFOIARequestsCollector from source_collectors.muckrock.schemas import MuckrockURLInfoSchema -from tests.test_automated.integration.core.helpers.constants import ALLEGHENY_COUNTY_MUCKROCK_ID, \ +from tests.automated.integration.core.helpers.constants import ALLEGHENY_COUNTY_MUCKROCK_ID, \ ALLEGHENY_COUNTY_TOWN_NAMES