diff --git a/alembic/versions/2025_12_18_0951-9292faed37fd_rename_suggestion_tables_to_consistent_.py b/alembic/versions/2025_12_18_0951-9292faed37fd_rename_suggestion_tables_to_consistent_.py new file mode 100644 index 00000000..fabfe098 --- /dev/null +++ b/alembic/versions/2025_12_18_0951-9292faed37fd_rename_suggestion_tables_to_consistent_.py @@ -0,0 +1,53 @@ +"""Rename suggestion tables to consistent nomenclature + +Revision ID: 9292faed37fd +Revises: dfb64594049f +Create Date: 2025-12-18 09:51:20.074946 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9292faed37fd' +down_revision: Union[str, None] = 'dfb64594049f' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +OLD_NEW_TABLE_MAPPING = { + # Anonymous Suggestions + "anonymous_annotation_agency": "annotation__agency__anon", + "anonymous_annotation_location": "annotation__location__anon", + "anonymous_annotation_record_type": "annotation__record_type__anon", + "anonymous_annotation_url_type": "annotation__url_type__anon", + # User Suggestions + "user_url_agency_suggestions": "annotation__agency__user", + "user_location_suggestions": "annotation__location__user", + "user_record_type_suggestions": "annotation__record_type__user", + "user_url_type_suggestions": "annotation__url_type__user", + # Auto suggestions + "auto_location_id_subtasks": "annotation__location__auto__subtasks", + "location_id_subtask_suggestions": "annotation__location__auto__suggestions", + "url_auto_agency_id_subtasks": "annotation__agency__auto__subtasks", + "agency_id_subtask_suggestions": "annotation__agency__auto__suggestions", + "auto_record_type_suggestions": "annotation__record_type__auto", + "auto_relevant_suggestions": "annotation__url_type__auto", + # Name suggestions + "url_name_suggestions": "annotation__name__suggestions", + "link__anonymous_sessions__name_suggestions": "annotation__name__anon__endorsements", + "link_user_name_suggestions": "annotation__name__user__endorsements", +} + +def upgrade() -> None: + for old_table_name, new_table_name in OLD_NEW_TABLE_MAPPING.items(): + op.rename_table( + old_table_name=old_table_name, + new_table_name=new_table_name + ) + + +def downgrade() -> None: + pass diff --git a/src/api/endpoints/annotate/_shared/extract.py b/src/api/endpoints/annotate/_shared/extract.py index 3fb7770b..12368cd6 100644 --- a/src/api/endpoints/annotate/_shared/extract.py +++ b/src/api/endpoints/annotate/_shared/extract.py @@ -16,8 +16,8 @@ from src.api.endpoints.annotate.all.get.queries.name.core import GetNameSuggestionsQueryBuilder from src.db.dto_converter import DTOConverter from src.db.dtos.url.mapping_.simple import SimpleURLMapping +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion async def extract_and_format_get_annotation_result( @@ -55,7 +55,7 @@ async def extract_and_format_get_annotation_result( batch_info=await GetAnnotationBatchInfoQueryBuilder( batch_id=batch_id, models=[ - UserURLAgencySuggestion, + AnnotationAgencyUser, ] ).run(session), location_suggestions=location_suggestions, diff --git a/src/api/endpoints/annotate/all/get/queries/agency/requester.py b/src/api/endpoints/annotate/all/get/queries/agency/requester.py index 9d933ae2..68d801b5 100644 --- a/src/api/endpoints/annotate/all/get/queries/agency/requester.py +++ b/src/api/endpoints/annotate/all/get/queries/agency/requester.py @@ -8,10 +8,10 @@ from src.db.helpers.query import exists_url from src.db.helpers.session import session_helper as sh from src.db.models.impl.agency.sqlalchemy import Agency +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser from src.db.models.impl.link.user_suggestion_not_found.agency.sqlalchemy import LinkUserSuggestionAgencyNotFound -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion from src.db.templates.requester import RequesterBase @@ -36,10 +36,10 @@ async def get_agency_suggestions(self) -> list[SuggestionModel]: .where( or_( exists_url( - UserURLAgencySuggestion + AnnotationAgencyUser ), exists_url( - URLAutoAgencyIDSubtask + AnnotationAgencyAutoSubtask ) ) ) @@ -49,13 +49,13 @@ async def get_agency_suggestions(self) -> list[SuggestionModel]: # Number of users who suggested each agency user_suggestions_cte = ( select( - UserURLAgencySuggestion.url_id, - UserURLAgencySuggestion.agency_id, - func.count(UserURLAgencySuggestion.user_id).label('user_count') + AnnotationAgencyUser.url_id, + AnnotationAgencyUser.agency_id, + func.count(AnnotationAgencyUser.user_id).label('user_count') ) .group_by( - UserURLAgencySuggestion.agency_id, - UserURLAgencySuggestion.url_id, + AnnotationAgencyUser.agency_id, + AnnotationAgencyUser.url_id, ) .cte("user_suggestions") ) @@ -63,20 +63,20 @@ async def get_agency_suggestions(self) -> list[SuggestionModel]: # Maximum confidence of robo annotation, if any robo_suggestions_cte = ( select( - URLAutoAgencyIDSubtask.url_id, + AnnotationAgencyAutoSubtask.url_id, Agency.id.label("agency_id"), - func.max(AgencyIDSubtaskSuggestion.confidence).label('robo_confidence') + func.max(AnnotationAgencyAutoSuggestion.confidence).label('robo_confidence') ) .join( - AgencyIDSubtaskSuggestion, - AgencyIDSubtaskSuggestion.subtask_id == URLAutoAgencyIDSubtask.id + AnnotationAgencyAutoSuggestion, + AnnotationAgencyAutoSuggestion.subtask_id == AnnotationAgencyAutoSubtask.id ) .join( Agency, - Agency.id == AgencyIDSubtaskSuggestion.agency_id + Agency.id == AnnotationAgencyAutoSuggestion.agency_id ) .group_by( - URLAutoAgencyIDSubtask.url_id, + AnnotationAgencyAutoSubtask.url_id, Agency.id ) .cte("robo_suggestions") diff --git a/src/api/endpoints/annotate/all/get/queries/convert.py b/src/api/endpoints/annotate/all/get/queries/convert.py index fe9b0777..0b0f0791 100644 --- a/src/api/endpoints/annotate/all/get/queries/convert.py +++ b/src/api/endpoints/annotate/all/get/queries/convert.py @@ -5,12 +5,12 @@ from src.api.endpoints.annotate.all.get.models.url_type import URLTypeAnnotationSuggestion from src.core.enums import RecordType from src.db.models.impl.flag.url_validated.enums import URLType -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType def convert_user_url_type_suggestion_to_url_type_annotation_suggestion( - db_suggestions: list[UserURLTypeSuggestion] + db_suggestions: list[AnnotationUserURLType] ) -> list[URLTypeAnnotationSuggestion]: counter: Counter[URLType] = Counter() for suggestion in db_suggestions: @@ -26,7 +26,7 @@ def convert_user_url_type_suggestion_to_url_type_annotation_suggestion( return anno_suggestions def convert_user_record_type_suggestion_to_record_type_annotation_suggestion( - db_suggestions: list[UserRecordTypeSuggestion] + db_suggestions: list[AnnotationUserRecordType] ) -> RecordTypeAnnotationResponseOuterInfo: counter: Counter[RecordType] = Counter() for suggestion in db_suggestions: diff --git a/src/api/endpoints/annotate/all/get/queries/core.py b/src/api/endpoints/annotate/all/get/queries/core.py index c63e8489..8f4fe7a9 100644 --- a/src/api/endpoints/annotate/all/get/queries/core.py +++ b/src/api/endpoints/annotate/all/get/queries/core.py @@ -5,13 +5,16 @@ from src.api.endpoints.annotate._shared.queries import helper from src.api.endpoints.annotate.all.get.models.response import GetNextURLForAllAnnotationResponse from src.collectors.enums import URLStatus +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser from src.db.models.impl.flag.url_suspended.sqlalchemy import FlagURLSuspended from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType +from src.db.models.views.unvalidated_url import UnvalidatedURL +from src.db.models.views.url_anno_count import URLAnnotationCount +from src.db.models.views.url_annotations_flags import URLAnnotationFlagsView from src.db.queries.base.builder import QueryBuilderBase @@ -45,35 +48,35 @@ async def run( URL.status == URLStatus.OK.value, # Must not have been previously annotated by user ~exists( - select(UserURLTypeSuggestion.url_id) + select(AnnotationUserURLType.url_id) .where( - UserURLTypeSuggestion.url_id == URL.id, - UserURLTypeSuggestion.user_id == self.user_id, + AnnotationUserURLType.url_id == URL.id, + AnnotationUserURLType.user_id == self.user_id, ) ), ~exists( - select(UserURLAgencySuggestion.url_id) + select(AnnotationAgencyUser.url_id) .where( - UserURLAgencySuggestion.url_id == URL.id, - UserURLAgencySuggestion.user_id == self.user_id, + AnnotationAgencyUser.url_id == URL.id, + AnnotationAgencyUser.user_id == self.user_id, ) ), ~exists( select( - UserLocationSuggestion.url_id + AnnotationLocationUser.url_id ) .where( - UserLocationSuggestion.url_id == URL.id, - UserLocationSuggestion.user_id == self.user_id, + AnnotationLocationUser.url_id == URL.id, + AnnotationLocationUser.user_id == self.user_id, ) ), ~exists( select( - UserRecordTypeSuggestion.url_id + AnnotationUserRecordType.url_id ) .where( - UserRecordTypeSuggestion.url_id == URL.id, - UserRecordTypeSuggestion.user_id == self.user_id, + AnnotationUserRecordType.url_id == URL.id, + AnnotationUserRecordType.user_id == self.user_id, ) ), ~exists( diff --git a/src/api/endpoints/annotate/all/get/queries/location_/requester.py b/src/api/endpoints/annotate/all/get/queries/location_/requester.py index fad8e834..49f00f89 100644 --- a/src/api/endpoints/annotate/all/get/queries/location_/requester.py +++ b/src/api/endpoints/annotate/all/get/queries/location_/requester.py @@ -6,10 +6,10 @@ from src.api.endpoints.annotate.all.get.queries._shared.sort import sort_suggestions from src.db.helpers.query import exists_url from src.db.helpers.session import session_helper as sh +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser from src.db.models.impl.link.user_suggestion_not_found.location.sqlalchemy import LinkUserSuggestionLocationNotFound -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion from src.db.models.views.location_expanded import LocationExpandedView from src.db.templates.requester import RequesterBase @@ -25,10 +25,10 @@ async def get_location_suggestions(self, url_id: int) -> list[SuggestionModel]: .where( or_( exists_url( - UserLocationSuggestion + AnnotationLocationUser ), exists_url( - AutoLocationIDSubtask + AnnotationLocationAutoSubtask ) ) ) @@ -37,34 +37,34 @@ async def get_location_suggestions(self, url_id: int) -> list[SuggestionModel]: # Number of users who suggested each location user_suggestions_cte = ( select( - UserLocationSuggestion.url_id, - UserLocationSuggestion.location_id, - func.count(UserLocationSuggestion.user_id).label('user_count') + AnnotationLocationUser.url_id, + AnnotationLocationUser.location_id, + func.count(AnnotationLocationUser.user_id).label('user_count') ) .group_by( - UserLocationSuggestion.location_id, - UserLocationSuggestion.url_id, + AnnotationLocationUser.location_id, + AnnotationLocationUser.url_id, ) .cte("user_suggestions") ) # Maximum confidence of robo annotation, if any robo_suggestions_cte = ( select( - AutoLocationIDSubtask.url_id, + AnnotationLocationAutoSubtask.url_id, LocationExpandedView.id.label("location_id"), - func.max(LocationIDSubtaskSuggestion.confidence).label('robo_confidence') + func.max(AnnotationLocationAutoSuggestion.confidence).label('robo_confidence') ) .join( LocationExpandedView, - LocationExpandedView.id == LocationIDSubtaskSuggestion.location_id + LocationExpandedView.id == AnnotationLocationAutoSuggestion.location_id ) .join( - AutoLocationIDSubtask, - AutoLocationIDSubtask.id == LocationIDSubtaskSuggestion.subtask_id + AnnotationLocationAutoSubtask, + AnnotationLocationAutoSubtask.id == AnnotationLocationAutoSuggestion.subtask_id ) .group_by( LocationExpandedView.id, - AutoLocationIDSubtask.url_id, + AnnotationLocationAutoSubtask.url_id, ) .cte("robo_suggestions") ) diff --git a/src/api/endpoints/annotate/all/get/queries/name/core.py b/src/api/endpoints/annotate/all/get/queries/name/core.py index 9438f14e..9eba70ee 100644 --- a/src/api/endpoints/annotate/all/get/queries/name/core.py +++ b/src/api/endpoints/annotate/all/get/queries/name/core.py @@ -5,9 +5,9 @@ from src.api.endpoints.annotate.all.get.models.name import NameAnnotationSuggestion, NameAnnotationResponseOuterInfo from src.db.helpers.session import session_helper as sh -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.queries.base.builder import QueryBuilderBase @@ -23,30 +23,30 @@ def __init__( async def run(self, session: AsyncSession) -> NameAnnotationResponseOuterInfo: query = ( select( - URLNameSuggestion.id.label('id'), - URLNameSuggestion.suggestion.label('display_name'), + AnnotationNameSuggestion.id.label('id'), + AnnotationNameSuggestion.suggestion.label('display_name'), func.count( LinkUserNameSuggestion.user_id ).label('user_count'), case( - (URLNameSuggestion.source == NameSuggestionSource.HTML_METADATA_TITLE, 1), + (AnnotationNameSuggestion.source == NameSuggestionSource.HTML_METADATA_TITLE, 1), else_=0 ).label("robo_count") ) .outerjoin( LinkUserNameSuggestion, - LinkUserNameSuggestion.suggestion_id == URLNameSuggestion.id, + LinkUserNameSuggestion.suggestion_id == AnnotationNameSuggestion.id, ) .where( - URLNameSuggestion.url_id == self.url_id, + AnnotationNameSuggestion.url_id == self.url_id, ) .group_by( - URLNameSuggestion.id, - URLNameSuggestion.suggestion, + AnnotationNameSuggestion.id, + AnnotationNameSuggestion.suggestion, ) .order_by( func.count(LinkUserNameSuggestion.user_id).desc(), - URLNameSuggestion.id.asc(), + AnnotationNameSuggestion.id.asc(), ) .limit(3) ) diff --git a/src/api/endpoints/annotate/all/post/requester.py b/src/api/endpoints/annotate/all/post/requester.py index 8834ff76..2034ecc1 100644 --- a/src/api/endpoints/annotate/all/post/requester.py +++ b/src/api/endpoints/annotate/all/post/requester.py @@ -2,16 +2,16 @@ from src.api.endpoints.annotate.all.post.models.name import AnnotationPostNameInfo from src.core.enums import RecordType +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion from src.db.models.impl.flag.url_validated.enums import URLType -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.models.impl.link.user_suggestion_not_found.agency.sqlalchemy import LinkUserSuggestionAgencyNotFound from src.db.models.impl.link.user_suggestion_not_found.location.sqlalchemy import LinkUserSuggestionLocationNotFound -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.templates.requester import RequesterBase @@ -33,7 +33,7 @@ def optionally_add_record_type( ) -> None: if rt is None: return - record_type_suggestion = UserRecordTypeSuggestion( + record_type_suggestion = AnnotationUserRecordType( url_id=self.url_id, user_id=self.user_id, record_type=rt.value @@ -44,7 +44,7 @@ def add_relevant_annotation( self, url_type: URLType, ) -> None: - relevant_suggestion = UserURLTypeSuggestion( + relevant_suggestion = AnnotationUserURLType( url_id=self.url_id, user_id=self.user_id, type=url_type @@ -53,7 +53,7 @@ def add_relevant_annotation( def add_agency_ids(self, agency_ids: list[int]) -> None: for agency_id in agency_ids: - agency_suggestion = UserURLAgencySuggestion( + agency_suggestion = AnnotationAgencyUser( url_id=self.url_id, user_id=self.user_id, agency_id=agency_id, @@ -61,9 +61,9 @@ def add_agency_ids(self, agency_ids: list[int]) -> None: self.session.add(agency_suggestion) def add_location_ids(self, location_ids: list[int]) -> None: - locations: list[UserLocationSuggestion] = [] + locations: list[AnnotationLocationUser] = [] for location_id in location_ids: - locations.append(UserLocationSuggestion( + locations.append(AnnotationLocationUser( url_id=self.url_id, user_id=self.user_id, location_id=location_id @@ -83,7 +83,7 @@ async def optionally_add_name_suggestion( ) self.session.add(link) return - name_suggestion = URLNameSuggestion( + name_suggestion = AnnotationNameSuggestion( url_id=self.url_id, suggestion=name_info.new_name, source=NameSuggestionSource.USER diff --git a/src/api/endpoints/annotate/anonymous/get/query.py b/src/api/endpoints/annotate/anonymous/get/query.py index ffe0a7b0..a7b96c1e 100644 --- a/src/api/endpoints/annotate/anonymous/get/query.py +++ b/src/api/endpoints/annotate/anonymous/get/query.py @@ -11,12 +11,12 @@ from src.api.endpoints.annotate.anonymous.get.response import GetNextURLForAnonymousAnnotationResponse from src.collectors.enums import URLStatus from src.db.helpers.query import not_exists_url +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType from src.db.models.impl.flag.url_suspended.sqlalchemy import FlagURLSuspended from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType from src.db.models.views.unvalidated_url import UnvalidatedURL from src.db.models.views.url_anno_count import URLAnnotationCount from src.db.models.views.url_annotations_flags import URLAnnotationFlagsView @@ -44,19 +44,19 @@ async def run(self, session: AsyncSession) -> GetNextURLForAnonymousAnnotationRe # Must not have been previously annotated by user not_exists_anon_annotation( session_id=self.session_id, - anon_model=AnonymousAnnotationURLType + anon_model=AnnotationAnonURLType ), not_exists_anon_annotation( session_id=self.session_id, - anon_model=AnonymousAnnotationRecordType + anon_model=AnnotationAnonRecordType ), not_exists_anon_annotation( session_id=self.session_id, - anon_model=AnonymousAnnotationLocation + anon_model=AnnotationLocationAnon ), not_exists_anon_annotation( session_id=self.session_id, - anon_model=AnonymousAnnotationAgency + anon_model=AnnotationAgencyAnon ), ~exists( select( diff --git a/src/api/endpoints/annotate/anonymous/post/query.py b/src/api/endpoints/annotate/anonymous/post/query.py index 29670c80..50ebad7c 100644 --- a/src/api/endpoints/annotate/anonymous/post/query.py +++ b/src/api/endpoints/annotate/anonymous/post/query.py @@ -3,13 +3,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from src.api.endpoints.annotate.all.post.models.request import AllAnnotationPostInfo -from src.db.models.impl.link.anonymous_sessions__name_suggestion import LinkAnonymousSessionNameSuggestion -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType +from src.db.models.impl.annotation.name.anon.sqlalchemy import AnnotationNameAnonEndorsement from src.db.queries.base.builder import QueryBuilderBase @@ -27,7 +27,7 @@ def __init__( async def run(self, session: AsyncSession) -> None: - url_type_suggestion = AnonymousAnnotationURLType( + url_type_suggestion = AnnotationAnonURLType( url_id=self.url_id, url_type=self.post_info.suggested_status, session_id=self.session_id @@ -36,7 +36,7 @@ async def run(self, session: AsyncSession) -> None: name_id: int | None if self.post_info.name_info.new_name is not None: - name_suggestion = URLNameSuggestion( + name_suggestion = AnnotationNameSuggestion( url_id=self.url_id, suggestion=self.post_info.name_info.new_name, source=NameSuggestionSource.USER @@ -50,14 +50,14 @@ async def run(self, session: AsyncSession) -> None: name_id = None if name_id is not None: - name_suggestion = LinkAnonymousSessionNameSuggestion( + name_suggestion = AnnotationNameAnonEndorsement( suggestion_id=name_id, session_id=self.session_id ) session.add(name_suggestion) if self.post_info.record_type is not None: - record_type_suggestion = AnonymousAnnotationRecordType( + record_type_suggestion = AnnotationAnonRecordType( url_id=self.url_id, record_type=self.post_info.record_type, session_id=self.session_id @@ -66,7 +66,7 @@ async def run(self, session: AsyncSession) -> None: if len(self.post_info.location_info.location_ids) != 0: location_suggestions = [ - AnonymousAnnotationLocation( + AnnotationLocationAnon( url_id=self.url_id, location_id=location_id, session_id=self.session_id @@ -77,7 +77,7 @@ async def run(self, session: AsyncSession) -> None: if len(self.post_info.agency_info.agency_ids) != 0: agency_suggestions = [ - AnonymousAnnotationAgency( + AnnotationAgencyAnon( url_id=self.url_id, agency_id=agency_id, session_id=self.session_id @@ -86,4 +86,3 @@ async def run(self, session: AsyncSession) -> None: ] session.add_all(agency_suggestions) - # Ignore Name suggestions \ No newline at end of file diff --git a/src/api/endpoints/contributions/shared/contributions.py b/src/api/endpoints/contributions/shared/contributions.py index ae72fc00..e62c0e7f 100644 --- a/src/api/endpoints/contributions/shared/contributions.py +++ b/src/api/endpoints/contributions/shared/contributions.py @@ -1,6 +1,6 @@ from sqlalchemy import select, func, CTE, Column -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType class ContributionsCTEContainer: @@ -8,11 +8,11 @@ class ContributionsCTEContainer: def __init__(self): self._cte = ( select( - UserURLTypeSuggestion.user_id, + AnnotationUserURLType.user_id, func.count().label("count") ) .group_by( - UserURLTypeSuggestion.user_id + AnnotationUserURLType.user_id ) .cte("contributions") ) diff --git a/src/api/endpoints/contributions/user/queries/agreement/agency.py b/src/api/endpoints/contributions/user/queries/agreement/agency.py index 01000bf2..c1dfeed7 100644 --- a/src/api/endpoints/contributions/user/queries/agreement/agency.py +++ b/src/api/endpoints/contributions/user/queries/agreement/agency.py @@ -1,14 +1,14 @@ from sqlalchemy import select, func, exists, and_, or_, any_, cast, Float from src.api.endpoints.contributions.user.queries.templates.agreement import AgreementCTEContainer +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion def get_agency_agreement_cte_container() -> AgreementCTEContainer: - uuas = UserURLAgencySuggestion + uuas = AnnotationAgencyUser fuv = FlagURLValidated lau = LinkURLAgency # CTE 1: All validated Meta URLs/Data Sources and their agencies diff --git a/src/api/endpoints/contributions/user/queries/agreement/record_type.py b/src/api/endpoints/contributions/user/queries/agreement/record_type.py index 2cde5ab5..278c4c60 100644 --- a/src/api/endpoints/contributions/user/queries/agreement/record_type.py +++ b/src/api/endpoints/contributions/user/queries/agreement/record_type.py @@ -3,7 +3,7 @@ from src.api.endpoints.contributions.user.queries.annotated_and_validated import AnnotatedAndValidatedCTEContainer from src.api.endpoints.contributions.user.queries.templates.agreement import AgreementCTEContainer from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType def get_record_type_agreement_cte_container( @@ -16,8 +16,8 @@ def get_record_type_agreement_cte_container( func.count() ) .join( - UserRecordTypeSuggestion, - UserRecordTypeSuggestion.url_id == inner_cte.url_id + AnnotationUserRecordType, + AnnotationUserRecordType.url_id == inner_cte.url_id ) .group_by( inner_cte.user_id @@ -31,14 +31,14 @@ def get_record_type_agreement_cte_container( func.count() ) .join( - UserRecordTypeSuggestion, - UserRecordTypeSuggestion.url_id == inner_cte.url_id + AnnotationUserRecordType, + AnnotationUserRecordType.url_id == inner_cte.url_id ) .join( URLRecordType, and_( URLRecordType.url_id == inner_cte.url_id, - URLRecordType.record_type == UserRecordTypeSuggestion.record_type + URLRecordType.record_type == AnnotationUserRecordType.record_type ) ) .group_by( diff --git a/src/api/endpoints/contributions/user/queries/agreement/url_type.py b/src/api/endpoints/contributions/user/queries/agreement/url_type.py index 12feb834..57a2a5a1 100644 --- a/src/api/endpoints/contributions/user/queries/agreement/url_type.py +++ b/src/api/endpoints/contributions/user/queries/agreement/url_type.py @@ -3,7 +3,7 @@ from src.api.endpoints.contributions.user.queries.annotated_and_validated import AnnotatedAndValidatedCTEContainer from src.api.endpoints.contributions.user.queries.templates.agreement import AgreementCTEContainer from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType def get_url_type_agreement_cte_container( @@ -17,8 +17,8 @@ def get_url_type_agreement_cte_container( func.count() ) .join( - UserURLTypeSuggestion, - UserURLTypeSuggestion.url_id == inner_cte.url_id + AnnotationUserURLType, + AnnotationUserURLType.url_id == inner_cte.url_id ) .join( FlagURLValidated, @@ -36,14 +36,14 @@ def get_url_type_agreement_cte_container( func.count() ) .join( - UserURLTypeSuggestion, - UserURLTypeSuggestion.url_id == inner_cte.url_id + AnnotationUserURLType, + AnnotationUserURLType.url_id == inner_cte.url_id ) .join( FlagURLValidated, and_( FlagURLValidated.url_id == inner_cte.url_id, - UserURLTypeSuggestion.type == FlagURLValidated.type + AnnotationUserURLType.type == FlagURLValidated.type ) ) diff --git a/src/api/endpoints/contributions/user/queries/annotated_and_validated.py b/src/api/endpoints/contributions/user/queries/annotated_and_validated.py index 9c7c48f6..1be14e28 100644 --- a/src/api/endpoints/contributions/user/queries/annotated_and_validated.py +++ b/src/api/endpoints/contributions/user/queries/annotated_and_validated.py @@ -1,7 +1,7 @@ from sqlalchemy import select, Column, CTE from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType class AnnotatedAndValidatedCTEContainer: @@ -9,16 +9,16 @@ class AnnotatedAndValidatedCTEContainer: def __init__(self, user_id: int | None): self._cte = ( select( - UserURLTypeSuggestion.user_id, - UserURLTypeSuggestion.url_id + AnnotationUserURLType.user_id, + AnnotationUserURLType.url_id ) .join( FlagURLValidated, - FlagURLValidated.url_id == UserURLTypeSuggestion.url_id + FlagURLValidated.url_id == AnnotationUserURLType.url_id ) ) if user_id is not None: - self._cte = self._cte.where(UserURLTypeSuggestion.user_id == user_id) + self._cte = self._cte.where(AnnotationUserURLType.user_id == user_id) self._cte = self._cte.cte("annotated_and_validated") @property diff --git a/src/api/endpoints/metrics/urls/breakdown/query/core.py b/src/api/endpoints/metrics/urls/breakdown/query/core.py index bccc7d68..949c8abd 100644 --- a/src/api/endpoints/metrics/urls/breakdown/query/core.py +++ b/src/api/endpoints/metrics/urls/breakdown/query/core.py @@ -1,16 +1,14 @@ -from typing import Any - from sqlalchemy import select, case, literal, func from sqlalchemy.ext.asyncio import AsyncSession from src.api.endpoints.metrics.dtos.get.urls.breakdown.pending import GetMetricsURLsBreakdownPendingResponseInnerDTO, \ GetMetricsURLsBreakdownPendingResponseDTO from src.collectors.enums import URLStatus +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.queries.base.builder import QueryBuilderBase @@ -21,19 +19,19 @@ async def run(self, session: AsyncSession) -> GetMetricsURLsBreakdownPendingResp flags = ( select( URL.id.label("url_id"), - case((UserRecordTypeSuggestion.url_id != None, literal(True)), else_=literal(False)).label( + case((AnnotationUserRecordType.url_id != None, literal(True)), else_=literal(False)).label( "has_user_record_type_annotation" ), - case((UserURLTypeSuggestion.url_id != None, literal(True)), else_=literal(False)).label( + case((AnnotationUserURLType.url_id != None, literal(True)), else_=literal(False)).label( "has_user_relevant_annotation" ), - case((UserURLAgencySuggestion.url_id != None, literal(True)), else_=literal(False)).label( + case((AnnotationAgencyUser.url_id != None, literal(True)), else_=literal(False)).label( "has_user_agency_annotation" ), ) - .outerjoin(UserRecordTypeSuggestion, URL.id == UserRecordTypeSuggestion.url_id) - .outerjoin(UserURLTypeSuggestion, URL.id == UserURLTypeSuggestion.url_id) - .outerjoin(UserURLAgencySuggestion, URL.id == UserURLAgencySuggestion.url_id) + .outerjoin(AnnotationUserRecordType, URL.id == AnnotationUserRecordType.url_id) + .outerjoin(AnnotationUserURLType, URL.id == AnnotationUserURLType.url_id) + .outerjoin(AnnotationAgencyUser, URL.id == AnnotationAgencyUser.url_id) ).cte("flags") month = func.date_trunc('month', URL.created_at) diff --git a/src/api/endpoints/submit/data_source/queries/core.py b/src/api/endpoints/submit/data_source/queries/core.py index 1f97cd11..17233386 100644 --- a/src/api/endpoints/submit/data_source/queries/core.py +++ b/src/api/endpoints/submit/data_source/queries/core.py @@ -8,18 +8,18 @@ from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest from src.collectors.enums import URLStatus from src.core.enums import BatchStatus +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType from src.db.models.impl.batch.sqlalchemy import Batch from src.db.models.impl.flag.url_validated.enums import URLType from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL from src.db.models.impl.url.core.enums import URLSource from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion from src.db.queries.base.builder import QueryBuilderBase from src.db.queries.implementations.anonymous_session import MakeAnonymousSessionQueryBuilder from src.util.models.full_url import FullURL @@ -75,7 +75,7 @@ async def run( session_id: uuid.UUID = await MakeAnonymousSessionQueryBuilder().run(session=session) # Add URL Type Suggestion - url_type_suggestion = AnonymousAnnotationURLType( + url_type_suggestion = AnnotationAnonURLType( url_id=url_id, url_type=URLType.DATA_SOURCE, session_id=session_id @@ -84,7 +84,7 @@ async def run( # Optionally add Record Type as suggestion if self.request.record_type is not None: - record_type_suggestion = AnonymousAnnotationRecordType( + record_type_suggestion = AnnotationAnonRecordType( url_id=url_id, record_type=self.request.record_type.value, session_id=session_id @@ -94,7 +94,7 @@ async def run( # Optionally add Agency ID suggestions if self.request.agency_ids is not None: agency_id_suggestions = [ - AnonymousAnnotationAgency( + AnnotationAgencyAnon( url_id=url_id, agency_id=agency_id, session_id=session_id @@ -106,7 +106,7 @@ async def run( # Optionally add Location ID suggestions if self.request.location_ids is not None: location_id_suggestions = [ - AnonymousAnnotationLocation( + AnnotationLocationAnon( url_id=url_id, location_id=location_id, session_id=session_id @@ -117,7 +117,7 @@ async def run( # Optionally add name suggestion if self.request.name is not None: - name_suggestion = URLNameSuggestion( + name_suggestion = AnnotationNameSuggestion( url_id=url_id, suggestion=self.request.name, source=NameSuggestionSource.USER diff --git a/src/api/endpoints/submit/url/queries/core.py b/src/api/endpoints/submit/url/queries/core.py index 0d2c1c84..ccbbc2c4 100644 --- a/src/api/endpoints/submit/url/queries/core.py +++ b/src/api/endpoints/submit/url/queries/core.py @@ -8,15 +8,15 @@ convert_duplicate_urls_to_url_response from src.api.endpoints.submit.url.queries.dedupe import DeduplicateURLQueryBuilder from src.collectors.enums import URLStatus -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL from src.db.models.impl.url.core.enums import URLSource from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType from src.db.queries.base.builder import QueryBuilderBase from src.util.models.url_and_scheme import URLAndScheme from src.util.url import clean_url, get_url_and_scheme, is_valid_url @@ -77,7 +77,7 @@ async def run(self, session: AsyncSession) -> URLSubmissionResponse: # Add record type as suggestion if exists if self.request.record_type is not None: - rec_sugg = UserRecordTypeSuggestion( + rec_sugg = AnnotationUserRecordType( user_id=self.user_id, url_id=url_insert.id, record_type=self.request.record_type.value @@ -86,7 +86,7 @@ async def run(self, session: AsyncSession) -> URLSubmissionResponse: # Add name as suggestion if exists if self.request.name is not None: - name_sugg = URLNameSuggestion( + name_sugg = AnnotationNameSuggestion( url_id=url_insert.id, suggestion=self.request.name, source=NameSuggestionSource.USER @@ -104,7 +104,7 @@ async def run(self, session: AsyncSession) -> URLSubmissionResponse: # Add location ID as suggestion if exists if self.request.location_id is not None: - loc_sugg = UserLocationSuggestion( + loc_sugg = AnnotationLocationUser( user_id=self.user_id, url_id=url_insert.id, location_id=self.request.location_id @@ -113,7 +113,7 @@ async def run(self, session: AsyncSession) -> URLSubmissionResponse: # Add agency ID as suggestion if exists if self.request.agency_id is not None: - agen_sugg = UserURLAgencySuggestion( + agen_sugg = AnnotationAgencyUser( user_id=self.user_id, url_id=url_insert.id, agency_id=self.request.agency_id diff --git a/src/core/tasks/url/operators/agency_identification/core.py b/src/core/tasks/url/operators/agency_identification/core.py index 7657ea0e..536e4fec 100644 --- a/src/core/tasks/url/operators/agency_identification/core.py +++ b/src/core/tasks/url/operators/agency_identification/core.py @@ -9,7 +9,7 @@ from src.core.tasks.url.operators.base import URLTaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType class AgencyIdentificationTaskOperator( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/convert.py b/src/core/tasks/url/operators/agency_identification/subtasks/convert.py index 5cead5d3..a7d4735d 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/convert.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/convert.py @@ -1,7 +1,7 @@ from src.core.tasks.url.operators.agency_identification.subtasks.models.subtask import AutoAgencyIDSubtaskData from src.core.tasks.url.operators.agency_identification.subtasks.models.suggestion import AgencySuggestion -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic def convert_agency_suggestions_to_subtask_data( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/flags/core.py b/src/core/tasks/url/operators/agency_identification/subtasks/flags/core.py index 41997322..4eaeaeaa 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/flags/core.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/flags/core.py @@ -2,7 +2,7 @@ from environs import Env from src.core.tasks.url.operators.agency_identification.subtasks.flags.mappings import SUBTASK_TO_ENV_FLAG -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType class SubtaskFlagger: diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/flags/mappings.py b/src/core/tasks/url/operators/agency_identification/subtasks/flags/mappings.py index dcc0b60c..cc45b123 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/flags/mappings.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/flags/mappings.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType SUBTASK_TO_ENV_FLAG: dict[AutoAgencyIDSubtaskType, str] = { AutoAgencyIDSubtaskType.HOMEPAGE_MATCH: "AGENCY_ID_HOMEPAGE_MATCH_FLAG", diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/batch_link/core.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/batch_link/core.py index 9e15996f..83d4d11a 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/batch_link/core.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/batch_link/core.py @@ -6,8 +6,8 @@ from src.core.tasks.url.operators.agency_identification.subtasks.models.suggestion import AgencySuggestion from src.core.tasks.url.operators.agency_identification.subtasks.templates.subtask import AgencyIDSubtaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic class AgencyBatchLinkSubtaskOperator(AgencyIDSubtaskOperatorBase): diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/ckan_/core.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/ckan_/core.py index 2603191a..275bb3c6 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/ckan_/core.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/ckan_/core.py @@ -13,7 +13,7 @@ from src.core.tasks.url.operators.agency_identification.subtasks.templates.subtask import \ AgencyIDSubtaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from src.external.pdap.client import PDAPClient diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/convert.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/convert.py index f4ba913e..186ed9ca 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/convert.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/convert.py @@ -2,9 +2,9 @@ GetHomepageMatchParams from src.core.tasks.url.operators.agency_identification.subtasks.impl.homepage_match_.models.mapping import \ SubtaskURLMapping -from src.db.models.impl.url.suggestion.agency.subtask.enum import SubtaskDetailCode, AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic -from src.db.models.impl.url.suggestion.agency.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic +from src.db.models.impl.annotation.agency.auto.subtask.enum import SubtaskDetailCode, AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic def convert_params_to_subtask_entries( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/core.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/core.py index f335cb3a..d072aa6d 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/core.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/core.py @@ -7,8 +7,8 @@ from src.core.tasks.url.operators.agency_identification.subtasks.impl.homepage_match_.queries.get import \ GetHomepageMatchSubtaskURLsQueryBuilder from src.core.tasks.url.operators.agency_identification.subtasks.templates.subtask import AgencyIDSubtaskOperatorBase -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic -from src.db.models.impl.url.suggestion.agency.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic class HomepageMatchSubtaskOperator( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/models/entry.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/models/entry.py index 6c65f9ad..989e1a7b 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/models/entry.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/models/entry.py @@ -1,6 +1,6 @@ from pydantic import BaseModel, Field -from src.db.models.impl.url.suggestion.agency.subtask.enum import SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.enum import SubtaskDetailCode class GetHomepageMatchParams(BaseModel): diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/multi_agency_case.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/multi_agency_case.py index edf9e601..9c1fca04 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/multi_agency_case.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/multi_agency_case.py @@ -1,8 +1,8 @@ -from sqlalchemy import CTE, select, literal +from sqlalchemy import select, literal from src.core.tasks.url.operators.agency_identification.subtasks.impl.homepage_match_.queries.ctes.consolidated import \ CONSOLIDATED_CTE -from src.db.models.impl.url.suggestion.agency.subtask.enum import SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.enum import SubtaskDetailCode MULTI_AGENCY_CASE_QUERY = ( select( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/single_agency_case.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/single_agency_case.py index 5778ecb6..31638d31 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/single_agency_case.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/ctes/single_agency_case.py @@ -2,7 +2,7 @@ from src.core.tasks.url.operators.agency_identification.subtasks.impl.homepage_match_.queries.ctes.consolidated import \ CONSOLIDATED_CTE -from src.db.models.impl.url.suggestion.agency.subtask.enum import SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.enum import SubtaskDetailCode SINGLE_AGENCY_CASE_QUERY = ( select( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/get.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/get.py index 10619531..05f7dd81 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/get.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/homepage_match_/queries/get.py @@ -10,7 +10,7 @@ from src.core.tasks.url.operators.agency_identification.subtasks.impl.homepage_match_.queries.ctes.single_agency_case import \ SINGLE_AGENCY_CASE_QUERY from src.db.helpers.session import session_helper as sh -from src.db.models.impl.url.suggestion.agency.subtask.enum import SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.enum import SubtaskDetailCode from src.db.queries.base.builder import QueryBuilderBase diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/muckrock_/core.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/muckrock_/core.py index 030139ad..dd77b94e 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/muckrock_/core.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/muckrock_/core.py @@ -16,8 +16,8 @@ from src.core.tasks.url.operators.agency_identification.subtasks.queries.match_agency import MatchAgencyQueryBuilder from src.core.tasks.url.operators.agency_identification.subtasks.templates.subtask import AgencyIDSubtaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic from src.external.pdap.client import PDAPClient diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/convert.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/convert.py index 2766bff0..a3b8bb0f 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/convert.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/convert.py @@ -2,8 +2,8 @@ NLPLocationMatchSubtaskInput from src.core.tasks.url.operators.agency_identification.subtasks.models.subtask import AutoAgencyIDSubtaskData from src.core.tasks.url.operators.agency_identification.subtasks.models.suggestion import AgencySuggestion -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic def convert_location_agency_mappings_to_subtask_data_list( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/query_/query.py b/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/query_/query.py index f0dcac94..94eb48aa 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/query_/query.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/impl/nlp_location_match_/query_/query.py @@ -8,9 +8,9 @@ NLPLocationMatchSubtaskInput, LocationAnnotationToAgencyIDMapping, LocationAnnotation from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.ctes.subtask.impl.nlp_location import \ NLP_LOCATION_CONTAINER +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion from src.db.models.impl.link.agency_location.sqlalchemy import LinkAgencyLocation -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion from src.db.queries.base.builder import QueryBuilderBase from src.db.helpers.session import session_helper as sh @@ -21,21 +21,21 @@ async def run(self, session: AsyncSession) -> list[NLPLocationMatchSubtaskInput] query = ( select( NLP_LOCATION_CONTAINER.url_id, - LocationIDSubtaskSuggestion.location_id, - LocationIDSubtaskSuggestion.confidence, + AnnotationLocationAutoSuggestion.location_id, + AnnotationLocationAutoSuggestion.confidence, LinkAgencyLocation.agency_id, ) .join( - AutoLocationIDSubtask, - AutoLocationIDSubtask.url_id == NLP_LOCATION_CONTAINER.url_id + AnnotationLocationAutoSubtask, + AnnotationLocationAutoSubtask.url_id == NLP_LOCATION_CONTAINER.url_id ) .join( - LocationIDSubtaskSuggestion, - LocationIDSubtaskSuggestion.subtask_id == AutoLocationIDSubtask.id + AnnotationLocationAutoSuggestion, + AnnotationLocationAutoSuggestion.subtask_id == AnnotationLocationAutoSubtask.id ) .join( LinkAgencyLocation, - LinkAgencyLocation.location_id == LocationIDSubtaskSuggestion.location_id + LinkAgencyLocation.location_id == AnnotationLocationAutoSuggestion.location_id ) .where( ~NLP_LOCATION_CONTAINER.entry_exists diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/loader.py b/src/core/tasks/url/operators/agency_identification/subtasks/loader.py index 24099540..fd14d34e 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/loader.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/loader.py @@ -10,7 +10,7 @@ NLPLocationMatchSubtaskOperator from src.core.tasks.url.operators.agency_identification.subtasks.templates.subtask import AgencyIDSubtaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from src.external.pdap.client import PDAPClient diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/models/subtask.py b/src/core/tasks/url/operators/agency_identification/subtasks/models/subtask.py index 7da0a8f5..99f7b2d9 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/models/subtask.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/models/subtask.py @@ -1,7 +1,7 @@ from pydantic import BaseModel from src.core.tasks.url.operators.agency_identification.subtasks.models.suggestion import AgencySuggestion -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic class AutoAgencyIDSubtaskData(BaseModel): diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/constants.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/constants.py index bea99266..38a8b44c 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/constants.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/constants.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType # Determines priority of subtasks, all else being equal. SUBTASK_HIERARCHY: list[AutoAgencyIDSubtaskType] = [ diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/core.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/core.py index 2b81d2de..ef90db7f 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/core.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/core.py @@ -7,7 +7,7 @@ SUBTASK_HIERARCHY_MAPPING from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.eligible_counts import \ ELIGIBLE_COUNTS_QUERY -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from src.db.queries.base.builder import QueryBuilderBase from src.db.helpers.session import session_helper as sh diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py index cfb92327..4c5aaa78 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py @@ -2,24 +2,24 @@ from src.core.tasks.url.operators._shared.container.subtask.exists import \ URLsSubtaskExistsCTEContainer +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion cte = ( select( URL.id ) .join( - URLAutoAgencyIDSubtask, - URLAutoAgencyIDSubtask.url_id == URL.id, + AnnotationAgencyAutoSubtask, + AnnotationAgencyAutoSubtask.url_id == URL.id, ) .join( - AgencyIDSubtaskSuggestion, - AgencyIDSubtaskSuggestion.subtask_id == URLAutoAgencyIDSubtask.id, + AnnotationAgencyAutoSuggestion, + AnnotationAgencyAutoSuggestion.subtask_id == AnnotationAgencyAutoSubtask.id, ) .where( - AgencyIDSubtaskSuggestion.confidence >= 95, + AnnotationAgencyAutoSuggestion.confidence >= 95, ) .cte("high_confidence_annotations_exists") ) diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/helpers.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/helpers.py index b06442ea..7f4aff78 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/helpers.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/helpers.py @@ -1,8 +1,8 @@ from sqlalchemy import ColumnElement, exists +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask def get_exists_subtask_query( @@ -11,8 +11,8 @@ def get_exists_subtask_query( return ( exists() .where( - URLAutoAgencyIDSubtask.url_id == URL.id, - URLAutoAgencyIDSubtask.type == subtask_type, + AnnotationAgencyAutoSubtask.url_id == URL.id, + AnnotationAgencyAutoSubtask.type == subtask_type, ) .label("subtask_entry_exists") ) \ No newline at end of file diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py index 42fcc02f..167262b8 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py @@ -3,10 +3,10 @@ from src.core.tasks.url.operators._shared.container.subtask.eligible import URLsSubtaskEligibleCTEContainer from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.ctes.subtask.helpers import \ get_exists_subtask_query +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from src.db.models.impl.link.agency_batch.sqlalchemy import LinkAgencyBatch from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType cte = ( select( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/ckan.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/ckan.py index 6b8ed9e8..052a5fb3 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/ckan.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/ckan.py @@ -4,10 +4,10 @@ from src.core.tasks.url.operators._shared.container.subtask.eligible import URLsSubtaskEligibleCTEContainer from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.ctes.subtask.helpers import \ get_exists_subtask_query +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from src.db.models.impl.batch.sqlalchemy import Batch from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType cte = ( select( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/homepage.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/homepage.py index 7daba916..7cc9a065 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/homepage.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/homepage.py @@ -5,8 +5,8 @@ CONSOLIDATED_CTE from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.ctes.subtask.helpers import \ get_exists_subtask_query +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType VALID_URL_FLAG = ( exists() diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/muckrock.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/muckrock.py index 9e267f66..5a83e029 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/muckrock.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/muckrock.py @@ -4,10 +4,10 @@ from src.core.tasks.url.operators._shared.container.subtask.eligible import URLsSubtaskEligibleCTEContainer from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.ctes.subtask.helpers import \ get_exists_subtask_query +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from src.db.models.impl.batch.sqlalchemy import Batch from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType cte = ( select( diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location.py index 7a15b67a..fb22379d 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location.py @@ -5,11 +5,11 @@ from src.core.tasks.url.operators._shared.container.subtask.eligible import URLsSubtaskEligibleCTEContainer from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.ctes.subtask.helpers import \ get_exists_subtask_query +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion from src.db.models.impl.link.agency_location.sqlalchemy import LinkAgencyLocation from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion cte = ( select( @@ -19,10 +19,10 @@ ) ) .join( - AutoLocationIDSubtask, + AnnotationLocationAutoSubtask, and_( - AutoLocationIDSubtask.url_id == URL.id, - AutoLocationIDSubtask.locations_found + AnnotationLocationAutoSubtask.url_id == URL.id, + AnnotationLocationAutoSubtask.locations_found ) ) .where( @@ -32,12 +32,12 @@ LinkAgencyLocation.location_id ) .join( - LocationIDSubtaskSuggestion, - LocationIDSubtaskSuggestion.location_id == LinkAgencyLocation.location_id, + AnnotationLocationAutoSuggestion, + AnnotationLocationAutoSuggestion.location_id == LinkAgencyLocation.location_id, ) .join( - AutoLocationIDSubtask, - AutoLocationIDSubtask.id == LocationIDSubtaskSuggestion.subtask_id, + AnnotationLocationAutoSubtask, + AnnotationLocationAutoSubtask.id == AnnotationLocationAutoSuggestion.subtask_id, ) ) diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/eligible_counts.py b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/eligible_counts.py index d3b7fe6b..79067aae 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/eligible_counts.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/queries/survey/queries/eligible_counts.py @@ -2,7 +2,7 @@ from src.core.tasks.url.operators.agency_identification.subtasks.queries.survey.queries.ctes.eligible import \ EligibleContainer -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType def sum_count(col: ColumnElement[bool], subtask_type: AutoAgencyIDSubtaskType) -> ColumnElement[int]: diff --git a/src/core/tasks/url/operators/agency_identification/subtasks/templates/subtask.py b/src/core/tasks/url/operators/agency_identification/subtasks/templates/subtask.py index 9335afcf..d88933eb 100644 --- a/src/core/tasks/url/operators/agency_identification/subtasks/templates/subtask.py +++ b/src/core/tasks/url/operators/agency_identification/subtasks/templates/subtask.py @@ -6,8 +6,8 @@ from src.core.tasks.url.operators.agency_identification.subtasks.models.subtask import AutoAgencyIDSubtaskData from src.db.client.async_ import AsyncDatabaseClient from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic -from src.db.models.impl.url.suggestion.agency.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic from src.db.models.impl.url.task_error.pydantic_.insert import URLTaskErrorPydantic from src.db.models.impl.url.task_error.pydantic_.small import URLTaskErrorSmall diff --git a/src/core/tasks/url/operators/auto_name/clean.py b/src/core/tasks/url/operators/auto_name/clean.py index 2e1820ab..9c745829 100644 --- a/src/core/tasks/url/operators/auto_name/clean.py +++ b/src/core/tasks/url/operators/auto_name/clean.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.location.auto.subtask.constants import MAX_SUGGESTION_LENGTH +from src.db.models.impl.annotation.location.auto.subtask.constants import MAX_SUGGESTION_LENGTH def clean_title(title: str) -> str: diff --git a/src/core/tasks/url/operators/auto_name/core.py b/src/core/tasks/url/operators/auto_name/core.py index 00af9838..b5702008 100644 --- a/src/core/tasks/url/operators/auto_name/core.py +++ b/src/core/tasks/url/operators/auto_name/core.py @@ -4,8 +4,8 @@ from src.core.tasks.url.operators.auto_name.queries.prereq import AutoNamePrerequisitesQueryBuilder from src.core.tasks.url.operators.base import URLTaskOperatorBase from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.pydantic import URLNameSuggestionPydantic +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.pydantic import URLNameSuggestionPydantic class AutoNameURLTaskOperator(URLTaskOperatorBase): diff --git a/src/core/tasks/url/operators/auto_name/queries/cte.py b/src/core/tasks/url/operators/auto_name/queries/cte.py index 1c7fc503..ff8a958b 100644 --- a/src/core/tasks/url/operators/auto_name/queries/cte.py +++ b/src/core/tasks/url/operators/auto_name/queries/cte.py @@ -2,10 +2,10 @@ from src.db.enums import URLHTMLContentType, TaskType from src.db.helpers.query import no_url_task_error +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.html.content.sqlalchemy import URLHTMLContent -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion class AutoNamePrerequisiteCTEContainer: @@ -24,11 +24,11 @@ def __init__(self): URLHTMLContent.content_type == URLHTMLContentType.TITLE.value, ~exists( select( - URLNameSuggestion.id + AnnotationNameSuggestion.id ) .where( - URLNameSuggestion.url_id == URL.id, - URLNameSuggestion.source == NameSuggestionSource.HTML_METADATA_TITLE.value, + AnnotationNameSuggestion.url_id == URL.id, + AnnotationNameSuggestion.source == NameSuggestionSource.HTML_METADATA_TITLE.value, ) ), no_url_task_error(TaskType.AUTO_NAME) diff --git a/src/core/tasks/url/operators/auto_relevant/core.py b/src/core/tasks/url/operators/auto_relevant/core.py index 3acff217..ea2a80d4 100644 --- a/src/core/tasks/url/operators/auto_relevant/core.py +++ b/src/core/tasks/url/operators/auto_relevant/core.py @@ -5,9 +5,8 @@ from src.core.tasks.url.operators.auto_relevant.sort import separate_success_and_error_subsets from src.core.tasks.url.operators.base import URLTaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.url_type.auto.pydantic.input import AutoRelevancyAnnotationInput from src.db.enums import TaskType -from src.db.models.impl.url.task_error.pydantic_.insert import URLTaskErrorPydantic +from src.db.models.impl.annotation.url_type.auto.pydantic.input import AutoRelevancyAnnotationInput from src.db.models.impl.url.task_error.pydantic_.small import URLTaskErrorSmall from src.external.huggingface.inference.client import HuggingFaceInferenceClient from src.external.huggingface.inference.models.input import BasicInput diff --git a/src/core/tasks/url/operators/auto_relevant/queries/cte.py b/src/core/tasks/url/operators/auto_relevant/queries/cte.py index c8b816fd..354e4bd5 100644 --- a/src/core/tasks/url/operators/auto_relevant/queries/cte.py +++ b/src/core/tasks/url/operators/auto_relevant/queries/cte.py @@ -6,7 +6,7 @@ from src.db.helpers.query import not_exists_url, no_url_task_error from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.html.compressed.sqlalchemy import URLCompressedHTML -from src.db.models.impl.url.suggestion.url_type.auto.sqlalchemy import AutoRelevantSuggestion +from src.db.models.impl.annotation.url_type.auto.sqlalchemy import AnnotationAutoURLType class AutoRelevantPrerequisitesCTEContainer: @@ -22,7 +22,7 @@ def __init__(self): ) .where( URL.status == URLStatus.OK.value, - not_exists_url(AutoRelevantSuggestion), + not_exists_url(AnnotationAutoURLType), no_url_task_error(TaskType.RELEVANCY) ).cte("auto_relevant_prerequisites") ) diff --git a/src/core/tasks/url/operators/auto_relevant/queries/get.py b/src/core/tasks/url/operators/auto_relevant/queries/get.py index b566bb42..1ed115fa 100644 --- a/src/core/tasks/url/operators/auto_relevant/queries/get.py +++ b/src/core/tasks/url/operators/auto_relevant/queries/get.py @@ -4,14 +4,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload -from src.collectors.enums import URLStatus from src.core.tasks.url.operators.auto_relevant.models.tdo import URLRelevantTDO from src.core.tasks.url.operators.auto_relevant.queries.cte import AutoRelevantPrerequisitesCTEContainer -from src.db.models.impl.url.html.compressed.sqlalchemy import URLCompressedHTML from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.url_type.auto.sqlalchemy import AutoRelevantSuggestion from src.db.queries.base.builder import QueryBuilderBase -from src.db.statement_composer import StatementComposer from src.db.utils.compression import decompress_html diff --git a/src/core/tasks/url/operators/location_id/core.py b/src/core/tasks/url/operators/location_id/core.py index 3833a80c..82f7df13 100644 --- a/src/core/tasks/url/operators/location_id/core.py +++ b/src/core/tasks/url/operators/location_id/core.py @@ -8,7 +8,7 @@ from src.core.tasks.url.operators.location_id.subtasks.templates.subtask import LocationIDSubtaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType class LocationIdentificationTaskOperator( diff --git a/src/core/tasks/url/operators/location_id/subtasks/flags/core.py b/src/core/tasks/url/operators/location_id/subtasks/flags/core.py index 1b6cb55c..21765643 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/flags/core.py +++ b/src/core/tasks/url/operators/location_id/subtasks/flags/core.py @@ -1,7 +1,7 @@ from environs import Env from src.core.tasks.url.operators.location_id.subtasks.flags.mappings import SUBTASK_TO_ENV_FLAG -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType class SubtaskFlagger: diff --git a/src/core/tasks/url/operators/location_id/subtasks/flags/mappings.py b/src/core/tasks/url/operators/location_id/subtasks/flags/mappings.py index 48f5d194..548c4f7b 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/flags/mappings.py +++ b/src/core/tasks/url/operators/location_id/subtasks/flags/mappings.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType SUBTASK_TO_ENV_FLAG: dict[LocationIDSubtaskType, str] = { LocationIDSubtaskType.NLP_LOCATION_FREQUENCY: "LOCATION_ID_NLP_LOCATION_MATCH_FLAG", diff --git a/src/core/tasks/url/operators/location_id/subtasks/impl/batch_link/core.py b/src/core/tasks/url/operators/location_id/subtasks/impl/batch_link/core.py index a85e572a..59a7faf8 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/impl/batch_link/core.py +++ b/src/core/tasks/url/operators/location_id/subtasks/impl/batch_link/core.py @@ -5,8 +5,8 @@ from src.core.tasks.url.operators.location_id.subtasks.models.suggestion import LocationSuggestion from src.core.tasks.url.operators.location_id.subtasks.templates.subtask import LocationIDSubtaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic class LocationBatchLinkSubtaskOperator(LocationIDSubtaskOperatorBase): diff --git a/src/core/tasks/url/operators/location_id/subtasks/impl/nlp_location_freq/processor/convert.py b/src/core/tasks/url/operators/location_id/subtasks/impl/nlp_location_freq/processor/convert.py index 8ec60b35..26b0ff32 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/impl/nlp_location_freq/processor/convert.py +++ b/src/core/tasks/url/operators/location_id/subtasks/impl/nlp_location_freq/processor/convert.py @@ -17,8 +17,8 @@ SearchSimilarLocationsResponse from src.core.tasks.url.operators.location_id.subtasks.models.subtask import AutoLocationIDSubtaskData from src.core.tasks.url.operators.location_id.subtasks.models.suggestion import LocationSuggestion -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic def convert_invalid_url_nlp_mappings_to_subtask_data_list( diff --git a/src/core/tasks/url/operators/location_id/subtasks/loader.py b/src/core/tasks/url/operators/location_id/subtasks/loader.py index 408b5a07..38ea8bed 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/loader.py +++ b/src/core/tasks/url/operators/location_id/subtasks/loader.py @@ -4,7 +4,7 @@ from src.core.tasks.url.operators.location_id.subtasks.impl.nlp_location_freq.processor.nlp.core import NLPProcessor from src.core.tasks.url.operators.location_id.subtasks.templates.subtask import LocationIDSubtaskOperatorBase from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType class LocationIdentificationSubtaskLoader: diff --git a/src/core/tasks/url/operators/location_id/subtasks/models/subtask.py b/src/core/tasks/url/operators/location_id/subtasks/models/subtask.py index b06d2ff9..fa935ecb 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/models/subtask.py +++ b/src/core/tasks/url/operators/location_id/subtasks/models/subtask.py @@ -1,7 +1,7 @@ from pydantic import BaseModel from src.core.tasks.url.operators.location_id.subtasks.models.suggestion import LocationSuggestion -from src.db.models.impl.url.suggestion.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic +from src.db.models.impl.annotation.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic class AutoLocationIDSubtaskData(BaseModel): diff --git a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/constants.py b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/constants.py index b9f85e2d..f3093b03 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/constants.py +++ b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/constants.py @@ -1,5 +1,5 @@ # Determines priority of subtasks, all else being equal. -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType SUBTASK_HIERARCHY: list[LocationIDSubtaskType] = [ LocationIDSubtaskType.NLP_LOCATION_FREQUENCY, diff --git a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/core.py b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/core.py index c267b89e..44cb0627 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/core.py +++ b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/core.py @@ -6,7 +6,7 @@ from src.core.tasks.url.operators.location_id.subtasks.queries.survey.constants import SUBTASK_HIERARCHY_MAPPING from src.core.tasks.url.operators.location_id.subtasks.queries.survey.queries.eligible_counts import \ ELIGIBLE_COUNTS_QUERY -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType from src.db.queries.base.builder import QueryBuilderBase from src.db.helpers.session import session_helper as sh diff --git a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py index 7d0dddfd..668e9e69 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py +++ b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/exists/high_confidence_annotations.py @@ -2,24 +2,24 @@ from src.core.tasks.url.operators._shared.container.subtask.exists import \ URLsSubtaskExistsCTEContainer +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion cte = ( select( URL.id ) .join( - AutoLocationIDSubtask, - AutoLocationIDSubtask.url_id == URL.id, + AnnotationLocationAutoSubtask, + AnnotationLocationAutoSubtask.url_id == URL.id, ) .join( - LocationIDSubtaskSuggestion, - LocationIDSubtaskSuggestion.subtask_id == AutoLocationIDSubtask.id, + AnnotationLocationAutoSuggestion, + AnnotationLocationAutoSuggestion.subtask_id == AnnotationLocationAutoSubtask.id, ) .where( - LocationIDSubtaskSuggestion.confidence >= 95, + AnnotationLocationAutoSuggestion.confidence >= 95, ) .cte("high_confidence_annotations_exists") ) diff --git a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/helpers.py b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/helpers.py index acd73c4b..54f114b8 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/helpers.py +++ b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/helpers.py @@ -1,8 +1,8 @@ from sqlalchemy import ColumnElement, exists +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask def get_exists_subtask_query( @@ -11,8 +11,8 @@ def get_exists_subtask_query( return ( exists() .where( - AutoLocationIDSubtask.url_id == URL.id, - AutoLocationIDSubtask.type == subtask_type, + AnnotationLocationAutoSubtask.url_id == URL.id, + AnnotationLocationAutoSubtask.type == subtask_type, ) .label("subtask_entry_exists") ) \ No newline at end of file diff --git a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py index 14c2f260..6d08cc76 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py +++ b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/batch_link.py @@ -3,10 +3,10 @@ from src.core.tasks.url.operators._shared.container.subtask.eligible import URLsSubtaskEligibleCTEContainer from src.core.tasks.url.operators.location_id.subtasks.queries.survey.queries.ctes.subtask.helpers import \ get_exists_subtask_query +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL from src.db.models.impl.link.location_batch.sqlalchemy import LinkLocationBatch from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType cte = ( select( diff --git a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location_freq.py b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location_freq.py index 7ab2e0eb..72b4cd81 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location_freq.py +++ b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/ctes/subtask/impl/nlp_location_freq.py @@ -3,9 +3,9 @@ from src.core.tasks.url.operators._shared.container.subtask.eligible import URLsSubtaskEligibleCTEContainer from src.core.tasks.url.operators.location_id.subtasks.queries.survey.queries.ctes.subtask.helpers import \ get_exists_subtask_query +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.html.compressed.sqlalchemy import URLCompressedHTML -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType cte = ( select( diff --git a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/eligible_counts.py b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/eligible_counts.py index b803b7f2..97c47a33 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/eligible_counts.py +++ b/src/core/tasks/url/operators/location_id/subtasks/queries/survey/queries/eligible_counts.py @@ -1,7 +1,7 @@ from sqlalchemy import ColumnElement, func, Integer, select from src.core.tasks.url.operators.location_id.subtasks.queries.survey.queries.ctes.eligible import EligibleContainer -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType def sum_count(col: ColumnElement[bool], subtask_type: LocationIDSubtaskType) -> ColumnElement[int]: diff --git a/src/core/tasks/url/operators/location_id/subtasks/templates/subtask.py b/src/core/tasks/url/operators/location_id/subtasks/templates/subtask.py index 8ee856c2..a5fb050b 100644 --- a/src/core/tasks/url/operators/location_id/subtasks/templates/subtask.py +++ b/src/core/tasks/url/operators/location_id/subtasks/templates/subtask.py @@ -7,8 +7,8 @@ from src.core.tasks.url.operators.location_id.subtasks.models.suggestion import LocationSuggestion from src.db.client.async_ import AsyncDatabaseClient from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic -from src.db.models.impl.url.suggestion.location.auto.suggestion.pydantic import LocationIDSubtaskSuggestionPydantic +from src.db.models.impl.annotation.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic +from src.db.models.impl.annotation.location.auto.suggestion.pydantic import LocationIDSubtaskSuggestionPydantic from src.db.models.impl.url.task_error.pydantic_.insert import URLTaskErrorPydantic from src.db.models.impl.url.task_error.pydantic_.small import URLTaskErrorSmall diff --git a/src/core/tasks/url/operators/record_type/core.py b/src/core/tasks/url/operators/record_type/core.py index 9f63a6a5..d6097ab0 100644 --- a/src/core/tasks/url/operators/record_type/core.py +++ b/src/core/tasks/url/operators/record_type/core.py @@ -7,7 +7,7 @@ from src.db.client.async_ import AsyncDatabaseClient from src.db.dtos.url.with_html import URLWithHTML from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.record_type.auto import AutoRecordTypeSuggestion +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType from src.db.models.impl.url.task_error.pydantic_.small import URLTaskErrorSmall @@ -72,9 +72,9 @@ async def put_results_into_database( record_type = tdo.record_type url_and_record_type_list.append((url_id, record_type)) # Add to database - suggestions: list[AutoRecordTypeSuggestion] = [] + suggestions: list[AnnotationAutoRecordType] = [] for url_id, record_type in url_and_record_type_list: - suggestion = AutoRecordTypeSuggestion( + suggestion = AnnotationAutoRecordType( url_id=url_id, record_type=record_type.value ) diff --git a/src/core/tasks/url/operators/record_type/queries/cte.py b/src/core/tasks/url/operators/record_type/queries/cte.py index 22d3db10..710dab03 100644 --- a/src/core/tasks/url/operators/record_type/queries/cte.py +++ b/src/core/tasks/url/operators/record_type/queries/cte.py @@ -4,7 +4,7 @@ from src.db.helpers.query import not_exists_url, no_url_task_error from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.html.compressed.sqlalchemy import URLCompressedHTML -from src.db.models.impl.url.suggestion.record_type.auto import AutoRecordTypeSuggestion +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType class RecordTypeTaskPrerequisiteCTEContainer: @@ -18,7 +18,7 @@ def __init__(self): URLCompressedHTML ) .where( - not_exists_url(AutoRecordTypeSuggestion), + not_exists_url(AnnotationAutoRecordType), no_url_task_error( TaskType.RECORD_TYPE ) diff --git a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/agency.py b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/agency.py index 36fe0a87..440e908a 100644 --- a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/agency.py +++ b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/agency.py @@ -2,31 +2,31 @@ from src.core.tasks.url.operators.validate.queries.ctes.counts.constants import ANONYMOUS_VOTE_RATIO from src.core.tasks.url.operators.validate.queries.ctes.counts.core import ValidatedCountsCTEContainer -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser from src.db.models.views.unvalidated_url import UnvalidatedURL _user_counts = ( select( - UserURLAgencySuggestion.url_id, - UserURLAgencySuggestion.agency_id.label("entity"), + AnnotationAgencyUser.url_id, + AnnotationAgencyUser.agency_id.label("entity"), func.count().label("votes") ) .group_by( - UserURLAgencySuggestion.url_id, - UserURLAgencySuggestion.agency_id + AnnotationAgencyUser.url_id, + AnnotationAgencyUser.agency_id ) ) _anon_counts = ( select( - AnonymousAnnotationAgency.url_id, - AnonymousAnnotationAgency.agency_id.label("entity"), + AnnotationAgencyAnon.url_id, + AnnotationAgencyAnon.agency_id.label("entity"), (func.count() / ANONYMOUS_VOTE_RATIO).label("votes") ) .group_by( - AnonymousAnnotationAgency.url_id, - AnonymousAnnotationAgency.agency_id + AnnotationAgencyAnon.url_id, + AnnotationAgencyAnon.agency_id ) ) diff --git a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/location.py b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/location.py index 4e180e18..496b14e1 100644 --- a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/location.py +++ b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/location.py @@ -2,32 +2,31 @@ from src.core.tasks.url.operators.validate.queries.ctes.counts.constants import ANONYMOUS_VOTE_RATIO from src.core.tasks.url.operators.validate.queries.ctes.counts.core import ValidatedCountsCTEContainer -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser from src.db.models.views.unvalidated_url import UnvalidatedURL _user_counts = ( select( - UserLocationSuggestion.url_id, - UserLocationSuggestion.location_id.label("entity"), + AnnotationLocationUser.url_id, + AnnotationLocationUser.location_id.label("entity"), func.count().label("votes") ) .group_by( - UserLocationSuggestion.url_id, - UserLocationSuggestion.location_id + AnnotationLocationUser.url_id, + AnnotationLocationUser.location_id ) ) _anon_counts = ( select( - AnonymousAnnotationLocation.url_id, - AnonymousAnnotationLocation.location_id.label("entity"), + AnnotationLocationAnon.url_id, + AnnotationLocationAnon.location_id.label("entity"), (func.count() / ANONYMOUS_VOTE_RATIO).label("votes") ) .group_by( - AnonymousAnnotationLocation.url_id, - AnonymousAnnotationLocation.location_id + AnnotationLocationAnon.url_id, + AnnotationLocationAnon.location_id ) ) diff --git a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/name.py b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/name.py index 4000e6e2..cec89ef2 100644 --- a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/name.py +++ b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/name.py @@ -1,41 +1,41 @@ from sqlalchemy import select, func from src.core.tasks.url.operators.validate.queries.ctes.counts.core import ValidatedCountsCTEContainer -from src.db.models.impl.link.anonymous_sessions__name_suggestion import LinkAnonymousSessionNameSuggestion -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.name.anon.sqlalchemy import AnnotationNameAnonEndorsement +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.models.views.unvalidated_url import UnvalidatedURL _user_counts = ( select( - URLNameSuggestion.url_id, - URLNameSuggestion.suggestion.label("entity"), + AnnotationNameSuggestion.url_id, + AnnotationNameSuggestion.suggestion.label("entity"), func.count().label("votes") ) .join( LinkUserNameSuggestion, - LinkUserNameSuggestion.suggestion_id == URLNameSuggestion.id + LinkUserNameSuggestion.suggestion_id == AnnotationNameSuggestion.id ) .group_by( - URLNameSuggestion.url_id, - URLNameSuggestion.suggestion + AnnotationNameSuggestion.url_id, + AnnotationNameSuggestion.suggestion ) .cte("user_counts") ) _anon_counts = ( select( - URLNameSuggestion.url_id, - URLNameSuggestion.suggestion.label("entity"), + AnnotationNameSuggestion.url_id, + AnnotationNameSuggestion.suggestion.label("entity"), func.count().label("votes") ) .join( - LinkAnonymousSessionNameSuggestion, - LinkAnonymousSessionNameSuggestion.suggestion_id == URLNameSuggestion.id + AnnotationNameAnonEndorsement, + AnnotationNameAnonEndorsement.suggestion_id == AnnotationNameSuggestion.id ) .group_by( - URLNameSuggestion.url_id, - URLNameSuggestion.suggestion + AnnotationNameSuggestion.url_id, + AnnotationNameSuggestion.suggestion ) .cte("anon_counts") ) diff --git a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/record_type.py b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/record_type.py index 65b1f9b0..efc92455 100644 --- a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/record_type.py +++ b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/record_type.py @@ -2,31 +2,31 @@ from src.core.tasks.url.operators.validate.queries.ctes.counts.constants import ANONYMOUS_VOTE_RATIO from src.core.tasks.url.operators.validate.queries.ctes.counts.core import ValidatedCountsCTEContainer -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType from src.db.models.views.unvalidated_url import UnvalidatedURL _user_counts = ( select( - UserRecordTypeSuggestion.url_id, - UserRecordTypeSuggestion.record_type.label("entity"), + AnnotationUserRecordType.url_id, + AnnotationUserRecordType.record_type.label("entity"), func.count().label("votes") ) .group_by( - UserRecordTypeSuggestion.url_id, - UserRecordTypeSuggestion.record_type + AnnotationUserRecordType.url_id, + AnnotationUserRecordType.record_type ) ) _anon_counts = ( select( - AnonymousAnnotationRecordType.url_id, - AnonymousAnnotationRecordType.record_type.label("entity"), + AnnotationAnonRecordType.url_id, + AnnotationAnonRecordType.record_type.label("entity"), (func.count() * ANONYMOUS_VOTE_RATIO).label("votes") ) .group_by( - AnonymousAnnotationRecordType.url_id, - AnonymousAnnotationRecordType.record_type + AnnotationAnonRecordType.url_id, + AnnotationAnonRecordType.record_type ) ) diff --git a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/url_type.py b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/url_type.py index 72638f19..6c87e69b 100644 --- a/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/url_type.py +++ b/src/core/tasks/url/operators/validate/queries/ctes/counts/impl/url_type.py @@ -2,31 +2,31 @@ from src.core.tasks.url.operators.validate.queries.ctes.counts.constants import ANONYMOUS_VOTE_RATIO from src.core.tasks.url.operators.validate.queries.ctes.counts.core import ValidatedCountsCTEContainer -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.models.views.unvalidated_url import UnvalidatedURL _user_counts = ( select( - UserURLTypeSuggestion.url_id, - UserURLTypeSuggestion.type.label("entity"), + AnnotationUserURLType.url_id, + AnnotationUserURLType.type.label("entity"), func.count().label("votes") ) .group_by( - UserURLTypeSuggestion.url_id, - UserURLTypeSuggestion.type + AnnotationUserURLType.url_id, + AnnotationUserURLType.type ) ) _anon_counts = ( select( - AnonymousAnnotationURLType.url_id, - AnonymousAnnotationURLType.url_type.label("entity"), + AnnotationAnonURLType.url_id, + AnnotationAnonURLType.url_type.label("entity"), (func.count() / ANONYMOUS_VOTE_RATIO).label("votes") ) .group_by( - AnonymousAnnotationURLType.url_id, - AnonymousAnnotationURLType.url_type + AnnotationAnonURLType.url_id, + AnnotationAnonURLType.url_type ) ) diff --git a/src/db/client/async_.py b/src/db/client/async_.py index 125c594e..c780f9d1 100644 --- a/src/db/client/async_.py +++ b/src/db/client/async_.py @@ -1,7 +1,6 @@ from datetime import datetime from functools import wraps from typing import Optional, Any, List -from uuid import UUID, uuid4 from sqlalchemy import select, func, Select, and_, update, Row, text from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker, AsyncEngine @@ -50,13 +49,14 @@ from src.db.client.types import UserSuggestionModel from src.db.config_manager import ConfigManager from src.db.constants import PLACEHOLDER_AGENCY_NAME -from src.db.dtos.url.html_content import URLHTMLContentInfo from src.db.dtos.url.insert import InsertURLsInfo from src.db.dtos.url.raw_html import RawHTMLInfo from src.db.enums import TaskType from src.db.helpers.session import session_helper as sh from src.db.models.impl.agency.enums import AgencyType, JurisdictionType from src.db.models.impl.agency.sqlalchemy import Agency +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.url_type.auto.pydantic.input import AutoRelevancyAnnotationInput from src.db.models.impl.backlog_snapshot import BacklogSnapshot from src.db.models.impl.batch.pydantic.info import BatchInfo from src.db.models.impl.batch.sqlalchemy import Batch @@ -75,15 +75,11 @@ from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.data_source.sqlalchemy import DSAppLinkDataSource from src.db.models.impl.url.html.compressed.sqlalchemy import URLCompressedHTML -from src.db.models.impl.url.html.content.sqlalchemy import URLHTMLContent from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.anonymous import AnonymousSession -from src.db.models.impl.url.suggestion.record_type.auto import AutoRecordTypeSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.auto.pydantic.input import AutoRelevancyAnnotationInput -from src.db.models.impl.url.suggestion.url_type.auto.sqlalchemy import AutoRelevantSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.auto.sqlalchemy import AnnotationAutoURLType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.models.impl.url.task_error.sqlalchemy import URLTaskError from src.db.models.impl.url.web_metadata.sqlalchemy import URLWebMetadata from src.db.models.templates_.base import Base @@ -231,7 +227,7 @@ async def add_user_relevant_suggestions( inputs: list[AutoRelevancyAnnotationInput] ): models = [ - AutoRelevantSuggestion( + AnnotationAutoURLType( url_id=input_.url_id, relevant=input_.is_relevant, confidence=input_.confidence, @@ -267,7 +263,7 @@ async def add_user_relevant_suggestion( ): prior_suggestion = await self.get_user_suggestion( session, - model=UserURLTypeSuggestion, + model=AnnotationUserURLType, user_id=user_id, url_id=url_id ) @@ -275,7 +271,7 @@ async def add_user_relevant_suggestion( prior_suggestion.type = suggested_status.value return - suggestion = UserURLTypeSuggestion( + suggestion = AnnotationUserURLType( url_id=url_id, user_id=user_id, type=suggested_status.value @@ -292,7 +288,7 @@ async def add_auto_record_type_suggestion( url_id: int, record_type: RecordType ): - suggestion = AutoRecordTypeSuggestion( + suggestion = AnnotationAutoRecordType( url_id=url_id, record_type=record_type.value ) @@ -308,7 +304,7 @@ async def add_user_record_type_suggestion( ): prior_suggestion = await self.get_user_suggestion( session, - model=UserRecordTypeSuggestion, + model=AnnotationUserRecordType, user_id=user_id, url_id=url_id ) @@ -316,7 +312,7 @@ async def add_user_record_type_suggestion( prior_suggestion.record_type = record_type.value return - suggestion = UserRecordTypeSuggestion( + suggestion = AnnotationUserRecordType( url_id=url_id, user_id=user_id, record_type=record_type.value @@ -570,7 +566,7 @@ async def add_agency_manual_suggestion( ) await session.merge(agency) - url_agency_suggestion = UserURLAgencySuggestion( + url_agency_suggestion = AnnotationAgencyUser( url_id=url_id, agency_id=agency_id, user_id=user_id, diff --git a/src/db/client/types.py b/src/db/client/types.py index e4f70301..8e3bff0d 100644 --- a/src/db/client/types.py +++ b/src/db/client/types.py @@ -1,5 +1,5 @@ -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType -UserSuggestionModel = UserURLTypeSuggestion or UserRecordTypeSuggestion or UserURLAgencySuggestion +UserSuggestionModel = AnnotationUserURLType or AnnotationUserRecordType or AnnotationAgencyUser diff --git a/src/db/constants.py b/src/db/constants.py index c8821e7e..87fd1f19 100644 --- a/src/db/constants.py +++ b/src/db/constants.py @@ -1,13 +1,13 @@ -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType PLACEHOLDER_AGENCY_NAME = "PLACEHOLDER_AGENCY_NAME" STANDARD_ROW_LIMIT = 100 USER_ANNOTATION_MODELS = [ - UserURLTypeSuggestion, - UserRecordTypeSuggestion, - UserURLAgencySuggestion + AnnotationUserURLType, + AnnotationUserRecordType, + AnnotationAgencyUser ] \ No newline at end of file diff --git a/src/db/dto_converter.py b/src/db/dto_converter.py index 4c91a353..4eb5a4cd 100644 --- a/src/db/dto_converter.py +++ b/src/db/dto_converter.py @@ -1,23 +1,19 @@ from collections import Counter -from src.api.endpoints.annotate.agency.get.dto import GetNextURLForAgencyAgencyInfo from src.api.endpoints.annotate.relevance.get.dto import RelevanceAnnotationResponseInfo -from src.api.endpoints.review.next.dto import FinalReviewAnnotationRelevantInfo, FinalReviewAnnotationRecordTypeInfo, \ - FinalReviewAnnotationAgencyInfo -from src.core.enums import RecordType, SuggestionType +from src.api.endpoints.review.next.dto import FinalReviewAnnotationRelevantInfo, FinalReviewAnnotationRecordTypeInfo +from src.core.enums import RecordType from src.core.tasks.url.operators.html.scraper.parser.dtos.response_html import ResponseHTMLInfo from src.core.tasks.url.operators.html.scraper.parser.mapping import ENUM_TO_ATTRIBUTE_MAPPING from src.db.dtos.url.html_content import URLHTMLContentInfo from src.db.dtos.url.with_html import URLWithHTML -from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.html.content.enums import HTMLContentType from src.db.models.impl.url.html.content.sqlalchemy import URLHTMLContent -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.record_type.auto import AutoRecordTypeSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.auto.sqlalchemy import AutoRelevantSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.auto.sqlalchemy import AnnotationAutoURLType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType class DTOConverter: @@ -28,8 +24,8 @@ class DTOConverter: @staticmethod def final_review_annotation_relevant_info( - user_suggestions: list[UserURLTypeSuggestion], - auto_suggestion: AutoRelevantSuggestion + user_suggestions: list[AnnotationUserURLType], + auto_suggestion: AnnotationAutoURLType ) -> FinalReviewAnnotationRelevantInfo: auto_value = RelevanceAnnotationResponseInfo( @@ -48,8 +44,8 @@ def final_review_annotation_relevant_info( @staticmethod def final_review_annotation_record_type_info( - user_suggestions: list[UserRecordTypeSuggestion], - auto_suggestion: AutoRecordTypeSuggestion + user_suggestions: list[AnnotationUserRecordType], + auto_suggestion: AnnotationAutoRecordType ): if auto_suggestion is None: diff --git a/src/db/models/impl/agency/sqlalchemy.py b/src/db/models/impl/agency/sqlalchemy.py index 9e99a0be..a6c9c1cf 100644 --- a/src/db/models/impl/agency/sqlalchemy.py +++ b/src/db/models/impl/agency/sqlalchemy.py @@ -27,8 +27,8 @@ class Agency( ) # Relationships - automated_suggestions = relationship("AgencyIDSubtaskSuggestion") - user_suggestions = relationship("UserURLAgencySuggestion", back_populates="agency") + automated_suggestions = relationship("AnnotationAgencyAutoSuggestion") + user_suggestions = relationship("AnnotationAgencyUser", back_populates="agency") confirmed_urls = relationship("LinkURLAgency", back_populates="agency") locations = relationship( diff --git a/src/db/models/impl/url/suggestion/README.md b/src/db/models/impl/annotation/README.md similarity index 100% rename from src/db/models/impl/url/suggestion/README.md rename to src/db/models/impl/annotation/README.md diff --git a/src/db/models/impl/link/user_name_suggestion/__init__.py b/src/db/models/impl/annotation/__init__.py similarity index 100% rename from src/db/models/impl/link/user_name_suggestion/__init__.py rename to src/db/models/impl/annotation/__init__.py diff --git a/src/db/models/impl/url/suggestion/__init__.py b/src/db/models/impl/annotation/agency/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/__init__.py rename to src/db/models/impl/annotation/agency/__init__.py diff --git a/src/db/models/impl/url/suggestion/agency/__init__.py b/src/db/models/impl/annotation/agency/anon/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/agency/__init__.py rename to src/db/models/impl/annotation/agency/anon/__init__.py diff --git a/src/db/models/impl/url/suggestion/anonymous/agency/sqlalchemy.py b/src/db/models/impl/annotation/agency/anon/sqlalchemy.py similarity index 83% rename from src/db/models/impl/url/suggestion/anonymous/agency/sqlalchemy.py rename to src/db/models/impl/annotation/agency/anon/sqlalchemy.py index 6f750289..a2da332b 100644 --- a/src/db/models/impl/url/suggestion/anonymous/agency/sqlalchemy.py +++ b/src/db/models/impl/annotation/agency/anon/sqlalchemy.py @@ -4,14 +4,14 @@ from src.db.models.templates_.base import Base -class AnonymousAnnotationAgency( +class AnnotationAgencyAnon( Base, URLDependentMixin, AgencyDependentMixin, CreatedAtMixin, AnonymousSessionMixin ): - __tablename__ = "anonymous_annotation_agency" + __tablename__ = "annotation__agency__anon" __table_args__ = ( PrimaryKeyConstraint("session_id", "url_id", "agency_id"), ) \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/agency/subtask/__init__.py b/src/db/models/impl/annotation/agency/auto/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/agency/subtask/__init__.py rename to src/db/models/impl/annotation/agency/auto/__init__.py diff --git a/src/db/models/impl/url/suggestion/agency/suggestion/__init__.py b/src/db/models/impl/annotation/agency/auto/subtask/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/agency/suggestion/__init__.py rename to src/db/models/impl/annotation/agency/auto/subtask/__init__.py diff --git a/src/db/models/impl/url/suggestion/agency/subtask/enum.py b/src/db/models/impl/annotation/agency/auto/subtask/enum.py similarity index 100% rename from src/db/models/impl/url/suggestion/agency/subtask/enum.py rename to src/db/models/impl/annotation/agency/auto/subtask/enum.py diff --git a/src/db/models/impl/url/suggestion/agency/subtask/pydantic.py b/src/db/models/impl/annotation/agency/auto/subtask/pydantic.py similarity index 61% rename from src/db/models/impl/url/suggestion/agency/subtask/pydantic.py rename to src/db/models/impl/annotation/agency/auto/subtask/pydantic.py index f2e9be57..4faee30d 100644 --- a/src/db/models/impl/url/suggestion/agency/subtask/pydantic.py +++ b/src/db/models/impl/annotation/agency/auto/subtask/pydantic.py @@ -1,5 +1,5 @@ -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask from src.db.models.templates_.base import Base from src.db.templates.markers.bulk.insert import BulkInsertableModel @@ -14,4 +14,4 @@ class URLAutoAgencyIDSubtaskPydantic(BulkInsertableModel): @classmethod def sa_model(cls) -> type_alias[Base]: - return URLAutoAgencyIDSubtask \ No newline at end of file + return AnnotationAgencyAutoSubtask \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/agency/subtask/sqlalchemy.py b/src/db/models/impl/annotation/agency/auto/subtask/sqlalchemy.py similarity index 76% rename from src/db/models/impl/url/suggestion/agency/subtask/sqlalchemy.py rename to src/db/models/impl/annotation/agency/auto/subtask/sqlalchemy.py index 7a297ef1..56383a6a 100644 --- a/src/db/models/impl/url/suggestion/agency/subtask/sqlalchemy.py +++ b/src/db/models/impl/annotation/agency/auto/subtask/sqlalchemy.py @@ -1,20 +1,20 @@ from sqlalchemy.orm import relationship, Mapped from src.db.models.helpers import enum_column -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode from src.db.models.mixins import URLDependentMixin, CreatedAtMixin, TaskDependentMixin from src.db.models.templates_.with_id import WithIDBase import sqlalchemy as sa -class URLAutoAgencyIDSubtask( +class AnnotationAgencyAutoSubtask( WithIDBase, URLDependentMixin, TaskDependentMixin, CreatedAtMixin ): - __tablename__ = "url_auto_agency_id_subtasks" + __tablename__ = "annotation__agency__auto__subtasks" type: Mapped[AutoAgencyIDSubtaskType] = enum_column( AutoAgencyIDSubtaskType, @@ -30,6 +30,6 @@ class URLAutoAgencyIDSubtask( ) suggestions = relationship( - "AgencyIDSubtaskSuggestion", + "AnnotationAgencyAutoSuggestion", cascade="all, delete-orphan" ) \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/anonymous/agency/__init__.py b/src/db/models/impl/annotation/agency/auto/suggestion/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/anonymous/agency/__init__.py rename to src/db/models/impl/annotation/agency/auto/suggestion/__init__.py diff --git a/src/db/models/impl/url/suggestion/agency/suggestion/pydantic.py b/src/db/models/impl/annotation/agency/auto/suggestion/pydantic.py similarity index 69% rename from src/db/models/impl/url/suggestion/agency/suggestion/pydantic.py rename to src/db/models/impl/annotation/agency/auto/suggestion/pydantic.py index 5a0fd2b8..1ec38502 100644 --- a/src/db/models/impl/url/suggestion/agency/suggestion/pydantic.py +++ b/src/db/models/impl/annotation/agency/auto/suggestion/pydantic.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from src.db.models.templates_.base import Base from src.db.templates.markers.bulk.insert import BulkInsertableModel @@ -13,4 +13,4 @@ class AgencyIDSubtaskSuggestionPydantic( @classmethod def sa_model(cls) -> type[Base]: """Defines the SQLAlchemy model.""" - return AgencyIDSubtaskSuggestion \ No newline at end of file + return AnnotationAgencyAutoSuggestion \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/agency/suggestion/sqlalchemy.py b/src/db/models/impl/annotation/agency/auto/suggestion/sqlalchemy.py similarity index 77% rename from src/db/models/impl/url/suggestion/agency/suggestion/sqlalchemy.py rename to src/db/models/impl/annotation/agency/auto/suggestion/sqlalchemy.py index 3f8b8186..5cb715a5 100644 --- a/src/db/models/impl/url/suggestion/agency/suggestion/sqlalchemy.py +++ b/src/db/models/impl/annotation/agency/auto/suggestion/sqlalchemy.py @@ -5,17 +5,17 @@ from src.db.models.templates_.with_id import WithIDBase -class AgencyIDSubtaskSuggestion( +class AnnotationAgencyAutoSuggestion( WithIDBase, CreatedAtMixin, AgencyDependentMixin, ): - __tablename__ = "agency_id_subtask_suggestions" + __tablename__ = "annotation__agency__auto__suggestions" subtask_id = sa.Column( sa.Integer, - sa.ForeignKey("url_auto_agency_id_subtasks.id"), + sa.ForeignKey("annotation__agency__auto__subtasks.id"), nullable=False ) confidence = sa.Column( diff --git a/src/db/models/impl/url/suggestion/anonymous/location/__init__.py b/src/db/models/impl/annotation/agency/user/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/anonymous/location/__init__.py rename to src/db/models/impl/annotation/agency/user/__init__.py diff --git a/src/db/models/impl/url/suggestion/agency/user.py b/src/db/models/impl/annotation/agency/user/sqlalchemy.py similarity index 79% rename from src/db/models/impl/url/suggestion/agency/user.py rename to src/db/models/impl/annotation/agency/user/sqlalchemy.py index 79fa933c..6b00e06c 100644 --- a/src/db/models/impl/url/suggestion/agency/user.py +++ b/src/db/models/impl/annotation/agency/user/sqlalchemy.py @@ -4,11 +4,10 @@ from src.db.models.helpers import get_agency_id_foreign_column from src.db.models.mixins import URLDependentMixin from src.db.models.templates_.base import Base -from src.db.models.templates_.with_id import WithIDBase -class UserURLAgencySuggestion(URLDependentMixin, Base): - __tablename__ = "user_url_agency_suggestions" +class AnnotationAgencyUser(URLDependentMixin, Base): + __tablename__ = "annotation__agency__user" __table_args__ = ( PrimaryKeyConstraint("agency_id", "url_id", "user_id"), ) diff --git a/src/db/models/impl/url/suggestion/anonymous/record_type/__init__.py b/src/db/models/impl/annotation/location/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/anonymous/record_type/__init__.py rename to src/db/models/impl/annotation/location/__init__.py diff --git a/src/db/models/impl/url/suggestion/anonymous/session/__init__.py b/src/db/models/impl/annotation/location/anon/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/anonymous/session/__init__.py rename to src/db/models/impl/annotation/location/anon/__init__.py diff --git a/src/db/models/impl/url/suggestion/anonymous/location/sqlalchemy.py b/src/db/models/impl/annotation/location/anon/sqlalchemy.py similarity index 82% rename from src/db/models/impl/url/suggestion/anonymous/location/sqlalchemy.py rename to src/db/models/impl/annotation/location/anon/sqlalchemy.py index 3e39810b..6855b021 100644 --- a/src/db/models/impl/url/suggestion/anonymous/location/sqlalchemy.py +++ b/src/db/models/impl/annotation/location/anon/sqlalchemy.py @@ -4,7 +4,7 @@ from src.db.models.templates_.base import Base -class AnonymousAnnotationLocation( +class AnnotationLocationAnon( Base, URLDependentMixin, LocationDependentMixin, @@ -12,7 +12,7 @@ class AnonymousAnnotationLocation( AnonymousSessionMixin ): - __tablename__ = "anonymous_annotation_location" + __tablename__ = "annotation__location__anon" __table_args__ = ( PrimaryKeyConstraint("session_id", "url_id", "location_id"), ) \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/anonymous/url_type/__init__.py b/src/db/models/impl/annotation/location/auto/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/anonymous/url_type/__init__.py rename to src/db/models/impl/annotation/location/auto/__init__.py diff --git a/src/db/models/impl/url/suggestion/location/__init__.py b/src/db/models/impl/annotation/location/auto/subtask/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/location/__init__.py rename to src/db/models/impl/annotation/location/auto/subtask/__init__.py diff --git a/src/db/models/impl/url/suggestion/location/auto/subtask/constants.py b/src/db/models/impl/annotation/location/auto/subtask/constants.py similarity index 100% rename from src/db/models/impl/url/suggestion/location/auto/subtask/constants.py rename to src/db/models/impl/annotation/location/auto/subtask/constants.py diff --git a/src/db/models/impl/url/suggestion/location/auto/subtask/enums.py b/src/db/models/impl/annotation/location/auto/subtask/enums.py similarity index 100% rename from src/db/models/impl/url/suggestion/location/auto/subtask/enums.py rename to src/db/models/impl/annotation/location/auto/subtask/enums.py diff --git a/src/db/models/impl/url/suggestion/location/auto/subtask/pydantic.py b/src/db/models/impl/annotation/location/auto/subtask/pydantic.py similarity index 60% rename from src/db/models/impl/url/suggestion/location/auto/subtask/pydantic.py rename to src/db/models/impl/annotation/location/auto/subtask/pydantic.py index 091a00b9..8bf8c1ed 100644 --- a/src/db/models/impl/url/suggestion/location/auto/subtask/pydantic.py +++ b/src/db/models/impl/annotation/location/auto/subtask/pydantic.py @@ -1,5 +1,5 @@ -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask from src.db.models.templates_.base import Base from src.db.templates.markers.bulk.insert import BulkInsertableModel @@ -16,4 +16,4 @@ class AutoLocationIDSubtaskPydantic( @classmethod def sa_model(cls) -> type[Base]: """Defines the SQLAlchemy model.""" - return AutoLocationIDSubtask \ No newline at end of file + return AnnotationLocationAutoSubtask \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/location/auto/subtask/sqlalchemy.py b/src/db/models/impl/annotation/location/auto/subtask/sqlalchemy.py similarity index 63% rename from src/db/models/impl/url/suggestion/location/auto/subtask/sqlalchemy.py rename to src/db/models/impl/annotation/location/auto/subtask/sqlalchemy.py index b7412d1e..61654851 100644 --- a/src/db/models/impl/url/suggestion/location/auto/subtask/sqlalchemy.py +++ b/src/db/models/impl/annotation/location/auto/subtask/sqlalchemy.py @@ -2,20 +2,20 @@ from sqlalchemy.orm import relationship, Mapped from src.db.models.helpers import enum_column -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion from src.db.models.mixins import CreatedAtMixin, TaskDependentMixin, URLDependentMixin from src.db.models.templates_.with_id import WithIDBase -class AutoLocationIDSubtask( +class AnnotationLocationAutoSubtask( WithIDBase, CreatedAtMixin, TaskDependentMixin, URLDependentMixin, ): - __tablename__ = 'auto_location_id_subtasks' + __tablename__ = 'annotation__location__auto__subtasks' locations_found = Column(Boolean(), nullable=False) type: Mapped[LocationIDSubtaskType] = enum_column( @@ -24,5 +24,5 @@ class AutoLocationIDSubtask( ) suggestions = relationship( - LocationIDSubtaskSuggestion + AnnotationLocationAutoSuggestion ) \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/location/auto/__init__.py b/src/db/models/impl/annotation/location/auto/suggestion/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/location/auto/__init__.py rename to src/db/models/impl/annotation/location/auto/suggestion/__init__.py diff --git a/src/db/models/impl/url/suggestion/location/auto/suggestion/pydantic.py b/src/db/models/impl/annotation/location/auto/suggestion/pydantic.py similarity index 68% rename from src/db/models/impl/url/suggestion/location/auto/suggestion/pydantic.py rename to src/db/models/impl/annotation/location/auto/suggestion/pydantic.py index 1ddc53d7..792e3bd4 100644 --- a/src/db/models/impl/url/suggestion/location/auto/suggestion/pydantic.py +++ b/src/db/models/impl/annotation/location/auto/suggestion/pydantic.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion from src.db.models.templates_.base import Base from src.db.templates.markers.bulk.insert import BulkInsertableModel @@ -12,4 +12,4 @@ class LocationIDSubtaskSuggestionPydantic(BulkInsertableModel): @classmethod def sa_model(cls) -> type[Base]: """Defines the SQLAlchemy model.""" - return LocationIDSubtaskSuggestion \ No newline at end of file + return AnnotationLocationAutoSuggestion \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/location/auto/suggestion/sqlalchemy.py b/src/db/models/impl/annotation/location/auto/suggestion/sqlalchemy.py similarity index 78% rename from src/db/models/impl/url/suggestion/location/auto/suggestion/sqlalchemy.py rename to src/db/models/impl/annotation/location/auto/suggestion/sqlalchemy.py index 0d5ea926..f76d9eef 100644 --- a/src/db/models/impl/url/suggestion/location/auto/suggestion/sqlalchemy.py +++ b/src/db/models/impl/annotation/location/auto/suggestion/sqlalchemy.py @@ -5,11 +5,11 @@ from src.db.models.templates_.base import Base -class LocationIDSubtaskSuggestion( +class AnnotationLocationAutoSuggestion( Base, ): - __tablename__ = 'location_id_subtask_suggestions' + __tablename__ = 'annotation__location__auto__suggestions' __table_args__ = ( PrimaryKeyConstraint( 'subtask_id', @@ -19,7 +19,7 @@ class LocationIDSubtaskSuggestion( ) subtask_id = Column( Integer, - ForeignKey('auto_location_id_subtasks.id'), + ForeignKey('annotation__location__auto__subtasks.id'), nullable=False, primary_key=True, ) diff --git a/src/db/models/impl/url/suggestion/location/auto/subtask/__init__.py b/src/db/models/impl/annotation/location/user/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/location/auto/subtask/__init__.py rename to src/db/models/impl/annotation/location/user/__init__.py diff --git a/src/db/models/impl/url/suggestion/location/user/pydantic.py b/src/db/models/impl/annotation/location/user/pydantic.py similarity index 70% rename from src/db/models/impl/url/suggestion/location/user/pydantic.py rename to src/db/models/impl/annotation/location/user/pydantic.py index 11f2218b..c3bdcf11 100644 --- a/src/db/models/impl/url/suggestion/location/user/pydantic.py +++ b/src/db/models/impl/annotation/location/user/pydantic.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser from src.db.models.templates_.base import Base from src.db.templates.markers.bulk.insert import BulkInsertableModel @@ -13,4 +13,4 @@ class UserLocationSuggestionPydantic( @classmethod def sa_model(cls) -> type[Base]: """Defines the SQLAlchemy model.""" - return UserLocationSuggestion + return AnnotationLocationUser diff --git a/src/db/models/impl/url/suggestion/location/user/sqlalchemy.py b/src/db/models/impl/annotation/location/user/sqlalchemy.py similarity index 88% rename from src/db/models/impl/url/suggestion/location/user/sqlalchemy.py rename to src/db/models/impl/annotation/location/user/sqlalchemy.py index 18ac3851..614912fd 100644 --- a/src/db/models/impl/url/suggestion/location/user/sqlalchemy.py +++ b/src/db/models/impl/annotation/location/user/sqlalchemy.py @@ -6,13 +6,13 @@ from src.db.models.templates_.base import Base -class UserLocationSuggestion( +class AnnotationLocationUser( Base, CreatedAtMixin, LocationDependentMixin, URLDependentMixin ): - __tablename__ = 'user_location_suggestions' + __tablename__ = 'annotation__location__user' __table_args__ = ( PrimaryKeyConstraint('url_id', 'location_id', 'user_id'), ) diff --git a/src/db/models/impl/url/suggestion/location/auto/suggestion/__init__.py b/src/db/models/impl/annotation/name/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/location/auto/suggestion/__init__.py rename to src/db/models/impl/annotation/name/__init__.py diff --git a/src/db/models/impl/url/suggestion/location/user/__init__.py b/src/db/models/impl/annotation/name/anon/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/location/user/__init__.py rename to src/db/models/impl/annotation/name/anon/__init__.py diff --git a/src/db/models/impl/link/anonymous_sessions__name_suggestion.py b/src/db/models/impl/annotation/name/anon/sqlalchemy.py similarity index 75% rename from src/db/models/impl/link/anonymous_sessions__name_suggestion.py rename to src/db/models/impl/annotation/name/anon/sqlalchemy.py index a5773bd7..8e24a515 100644 --- a/src/db/models/impl/link/anonymous_sessions__name_suggestion.py +++ b/src/db/models/impl/annotation/name/anon/sqlalchemy.py @@ -4,15 +4,15 @@ from src.db.models.templates_.base import Base -class LinkAnonymousSessionNameSuggestion( +class AnnotationNameAnonEndorsement( Base, AnonymousSessionMixin, CreatedAtMixin ): - __tablename__ = "link__anonymous_sessions__name_suggestions" + __tablename__ = "annotation__name__anon__endorsements" suggestion_id = Column( Integer, - ForeignKey("url_name_suggestions.id"), + ForeignKey("annotation__name__suggestions.id"), primary_key=True, nullable=False, ) diff --git a/src/db/models/impl/url/suggestion/name/__init__.py b/src/db/models/impl/annotation/name/suggestion/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/name/__init__.py rename to src/db/models/impl/annotation/name/suggestion/__init__.py diff --git a/src/db/models/impl/url/suggestion/name/enums.py b/src/db/models/impl/annotation/name/suggestion/enums.py similarity index 100% rename from src/db/models/impl/url/suggestion/name/enums.py rename to src/db/models/impl/annotation/name/suggestion/enums.py diff --git a/src/db/models/impl/annotation/name/suggestion/pydantic.py b/src/db/models/impl/annotation/name/suggestion/pydantic.py new file mode 100644 index 00000000..55423a0a --- /dev/null +++ b/src/db/models/impl/annotation/name/suggestion/pydantic.py @@ -0,0 +1,17 @@ +from pydantic import Field + +from src.db.models.impl.annotation.location.auto.subtask.constants import MAX_SUGGESTION_LENGTH +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.templates.markers.bulk.insert import BulkInsertableModel + + +class URLNameSuggestionPydantic(BulkInsertableModel): + + url_id: int + suggestion: str = Field(..., max_length=MAX_SUGGESTION_LENGTH) + source: NameSuggestionSource + + @classmethod + def sa_model(cls) -> type[AnnotationNameSuggestion]: + return AnnotationNameSuggestion \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/name/sqlalchemy.py b/src/db/models/impl/annotation/name/suggestion/sqlalchemy.py similarity index 65% rename from src/db/models/impl/url/suggestion/name/sqlalchemy.py rename to src/db/models/impl/annotation/name/suggestion/sqlalchemy.py index 2f11542d..5aeee478 100644 --- a/src/db/models/impl/url/suggestion/name/sqlalchemy.py +++ b/src/db/models/impl/annotation/name/suggestion/sqlalchemy.py @@ -2,19 +2,19 @@ from sqlalchemy.orm import Mapped from src.db.models.helpers import enum_column -from src.db.models.impl.url.suggestion.location.auto.subtask.constants import MAX_SUGGESTION_LENGTH -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource +from src.db.models.impl.annotation.location.auto.subtask.constants import MAX_SUGGESTION_LENGTH +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource from src.db.models.mixins import URLDependentMixin, CreatedAtMixin from src.db.models.templates_.with_id import WithIDBase -class URLNameSuggestion( +class AnnotationNameSuggestion( WithIDBase, CreatedAtMixin, URLDependentMixin ): - __tablename__ = "url_name_suggestions" + __tablename__ = "annotation__name__suggestions" suggestion = Column(String(MAX_SUGGESTION_LENGTH), nullable=False) source: Mapped[NameSuggestionSource] = enum_column( diff --git a/src/db/models/impl/url/suggestion/record_type/__init__.py b/src/db/models/impl/annotation/name/user/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/record_type/__init__.py rename to src/db/models/impl/annotation/name/user/__init__.py diff --git a/src/db/models/impl/link/user_name_suggestion/pydantic.py b/src/db/models/impl/annotation/name/user/pydantic.py similarity index 75% rename from src/db/models/impl/link/user_name_suggestion/pydantic.py rename to src/db/models/impl/annotation/name/user/pydantic.py index 6e07989b..95fe0150 100644 --- a/src/db/models/impl/link/user_name_suggestion/pydantic.py +++ b/src/db/models/impl/annotation/name/user/pydantic.py @@ -1,4 +1,4 @@ -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.templates.markers.bulk.insert import BulkInsertableModel diff --git a/src/db/models/impl/link/user_name_suggestion/sqlalchemy.py b/src/db/models/impl/annotation/name/user/sqlalchemy.py similarity index 78% rename from src/db/models/impl/link/user_name_suggestion/sqlalchemy.py rename to src/db/models/impl/annotation/name/user/sqlalchemy.py index 316a8e3c..cf23dd6f 100644 --- a/src/db/models/impl/link/user_name_suggestion/sqlalchemy.py +++ b/src/db/models/impl/annotation/name/user/sqlalchemy.py @@ -9,11 +9,11 @@ class LinkUserNameSuggestion( CreatedAtMixin, ): - __tablename__ = "link_user_name_suggestions" + __tablename__ = "annotation__name__user__endorsements" suggestion_id = Column( Integer, - ForeignKey("url_name_suggestions.id"), + ForeignKey("annotation__name__suggestions.id"), primary_key=True, nullable=False, ) diff --git a/src/db/models/impl/url/suggestion/url_type/__init__.py b/src/db/models/impl/annotation/record_type/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/url_type/__init__.py rename to src/db/models/impl/annotation/record_type/__init__.py diff --git a/src/db/models/impl/url/suggestion/url_type/auto/__init__.py b/src/db/models/impl/annotation/record_type/anon/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/url_type/auto/__init__.py rename to src/db/models/impl/annotation/record_type/anon/__init__.py diff --git a/src/db/models/impl/url/suggestion/anonymous/record_type/sqlalchemy.py b/src/db/models/impl/annotation/record_type/anon/sqlalchemy.py similarity index 86% rename from src/db/models/impl/url/suggestion/anonymous/record_type/sqlalchemy.py rename to src/db/models/impl/annotation/record_type/anon/sqlalchemy.py index 22f37839..304ab1be 100644 --- a/src/db/models/impl/url/suggestion/anonymous/record_type/sqlalchemy.py +++ b/src/db/models/impl/annotation/record_type/anon/sqlalchemy.py @@ -7,13 +7,13 @@ from src.db.models.templates_.base import Base -class AnonymousAnnotationRecordType( +class AnnotationAnonRecordType( Base, URLDependentMixin, CreatedAtMixin, AnonymousSessionMixin ): - __tablename__ = "anonymous_annotation_record_type" + __tablename__ = "annotation__record_type__anon" __table_args__ = ( PrimaryKeyConstraint("session_id", "url_id", "record_type"), ) diff --git a/src/db/models/impl/url/suggestion/url_type/auto/pydantic/__init__.py b/src/db/models/impl/annotation/record_type/auto/__init__.py similarity index 100% rename from src/db/models/impl/url/suggestion/url_type/auto/pydantic/__init__.py rename to src/db/models/impl/annotation/record_type/auto/__init__.py diff --git a/src/db/models/impl/url/suggestion/record_type/auto.py b/src/db/models/impl/annotation/record_type/auto/sqlalchemy.py similarity index 89% rename from src/db/models/impl/url/suggestion/record_type/auto.py rename to src/db/models/impl/annotation/record_type/auto/sqlalchemy.py index 1c2c68d1..b09f01d8 100644 --- a/src/db/models/impl/url/suggestion/record_type/auto.py +++ b/src/db/models/impl/annotation/record_type/auto/sqlalchemy.py @@ -8,13 +8,13 @@ from src.db.models.types import record_type_values -class AutoRecordTypeSuggestion( +class AnnotationAutoRecordType( UpdatedAtMixin, CreatedAtMixin, URLDependentMixin, Base, ): - __tablename__ = "auto_record_type_suggestions" + __tablename__ = "annotation__record_type__auto" record_type = Column(postgresql.ENUM(*record_type_values, name='record_type'), nullable=False) __table_args__ = ( diff --git a/src/db/models/impl/annotation/record_type/user/__init__.py b/src/db/models/impl/annotation/record_type/user/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db/models/impl/url/suggestion/record_type/user.py b/src/db/models/impl/annotation/record_type/user/user.py similarity index 90% rename from src/db/models/impl/url/suggestion/record_type/user.py rename to src/db/models/impl/annotation/record_type/user/user.py index 4e271225..689d985b 100644 --- a/src/db/models/impl/url/suggestion/record_type/user.py +++ b/src/db/models/impl/annotation/record_type/user/user.py @@ -8,13 +8,13 @@ from src.db.models.types import record_type_values -class UserRecordTypeSuggestion( +class AnnotationUserRecordType( UpdatedAtMixin, CreatedAtMixin, URLDependentMixin, Base, ): - __tablename__ = "user_record_type_suggestions" + __tablename__ = "annotation__record_type__user" __table_args__ = ( PrimaryKeyConstraint("url_id", "user_id"), ) diff --git a/src/db/models/impl/annotation/url_type/__init__.py b/src/db/models/impl/annotation/url_type/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db/models/impl/annotation/url_type/anon/__init__.py b/src/db/models/impl/annotation/url_type/anon/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db/models/impl/url/suggestion/anonymous/url_type/sqlalchemy.py b/src/db/models/impl/annotation/url_type/anon/sqlalchemy.py similarity index 87% rename from src/db/models/impl/url/suggestion/anonymous/url_type/sqlalchemy.py rename to src/db/models/impl/annotation/url_type/anon/sqlalchemy.py index f0cbc6a7..a1de1826 100644 --- a/src/db/models/impl/url/suggestion/anonymous/url_type/sqlalchemy.py +++ b/src/db/models/impl/annotation/url_type/anon/sqlalchemy.py @@ -7,13 +7,13 @@ from src.db.models.templates_.base import Base -class AnonymousAnnotationURLType( +class AnnotationAnonURLType( Base, URLDependentMixin, CreatedAtMixin, AnonymousSessionMixin ): - __tablename__ = "anonymous_annotation_url_type" + __tablename__ = "annotation__url_type__anon" __table_args__ = ( PrimaryKeyConstraint("session_id", "url_id", "url_type"), ) diff --git a/src/db/models/impl/annotation/url_type/auto/__init__.py b/src/db/models/impl/annotation/url_type/auto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db/models/impl/annotation/url_type/auto/pydantic/__init__.py b/src/db/models/impl/annotation/url_type/auto/pydantic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db/models/impl/url/suggestion/url_type/auto/pydantic/input.py b/src/db/models/impl/annotation/url_type/auto/pydantic/input.py similarity index 100% rename from src/db/models/impl/url/suggestion/url_type/auto/pydantic/input.py rename to src/db/models/impl/annotation/url_type/auto/pydantic/input.py diff --git a/src/db/models/impl/url/suggestion/url_type/auto/sqlalchemy.py b/src/db/models/impl/annotation/url_type/auto/sqlalchemy.py similarity index 90% rename from src/db/models/impl/url/suggestion/url_type/auto/sqlalchemy.py rename to src/db/models/impl/annotation/url_type/auto/sqlalchemy.py index 19b5dc09..cc5fb7b8 100644 --- a/src/db/models/impl/url/suggestion/url_type/auto/sqlalchemy.py +++ b/src/db/models/impl/annotation/url_type/auto/sqlalchemy.py @@ -6,13 +6,13 @@ from src.db.models.templates_.with_id import WithIDBase -class AutoRelevantSuggestion( +class AnnotationAutoURLType( UpdatedAtMixin, CreatedAtMixin, URLDependentMixin, Base, ): - __tablename__ = "auto_relevant_suggestions" + __tablename__ = "annotation__url_type__auto" relevant = Column(Boolean, nullable=True) confidence = Column(Float, nullable=True) diff --git a/src/db/models/impl/annotation/url_type/user/__init__.py b/src/db/models/impl/annotation/url_type/user/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db/models/impl/url/suggestion/url_type/user.py b/src/db/models/impl/annotation/url_type/user/sqlalchemy.py similarity index 91% rename from src/db/models/impl/url/suggestion/url_type/user.py rename to src/db/models/impl/annotation/url_type/user/sqlalchemy.py index 52bbc4eb..af84a758 100644 --- a/src/db/models/impl/url/suggestion/url_type/user.py +++ b/src/db/models/impl/annotation/url_type/user/sqlalchemy.py @@ -9,13 +9,13 @@ from src.db.models.templates_.with_id import WithIDBase -class UserURLTypeSuggestion( +class AnnotationUserURLType( UpdatedAtMixin, CreatedAtMixin, URLDependentMixin, Base, ): - __tablename__ = "user_url_type_suggestions" + __tablename__ = "annotation__url_type__user" __table_args__ = ( PrimaryKeyConstraint("url_id", "user_id"), ) diff --git a/src/db/models/impl/anon_session/__init__.py b/src/db/models/impl/anon_session/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db/models/impl/url/suggestion/anonymous/session/sqlalchemy.py b/src/db/models/impl/anon_session/sqlalchemy.py similarity index 100% rename from src/db/models/impl/url/suggestion/anonymous/session/sqlalchemy.py rename to src/db/models/impl/anon_session/sqlalchemy.py diff --git a/src/db/models/impl/url/core/sqlalchemy.py b/src/db/models/impl/url/core/sqlalchemy.py index de4af177..dd52c1e1 100644 --- a/src/db/models/impl/url/core/sqlalchemy.py +++ b/src/db/models/impl/url/core/sqlalchemy.py @@ -1,18 +1,23 @@ from sqlalchemy import Column, Text, String, JSON, case, literal, Boolean from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship, Mapped -from sqlalchemy.util import hybridproperty from src.collectors.enums import URLStatus from src.db.models.helpers import enum_column +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion from src.db.models.impl.link.user_suggestion_not_found.location.sqlalchemy import LinkUserSuggestionLocationNotFound from src.db.models.impl.url.checked_for_duplicate import URLCheckedForDuplicate from src.db.models.impl.url.core.enums import URLSource from src.db.models.impl.url.html.compressed.sqlalchemy import URLCompressedHTML from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.auto.sqlalchemy import AnnotationAutoURLType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.models.impl.url.task_error.sqlalchemy import URLTaskError from src.db.models.mixins import UpdatedAtMixin, CreatedAtMixin from src.db.models.templates_.with_id import WithIDBase @@ -90,35 +95,35 @@ def full_url(cls): name_suggestions = relationship( - URLNameSuggestion + AnnotationNameSuggestion ) # Location user_location_suggestions = relationship( - UserLocationSuggestion + AnnotationLocationUser ) user_location_suggestion_not_found = relationship( LinkUserSuggestionLocationNotFound ) auto_location_subtasks = relationship( - AutoLocationIDSubtask + AnnotationLocationAutoSubtask ) # Agency user_agency_suggestions = relationship( - "UserURLAgencySuggestion", back_populates="url") + AnnotationAgencyUser, back_populates="url") auto_agency_subtasks = relationship( - "URLAutoAgencyIDSubtask" + AnnotationAgencyAutoSubtask ) # Record Type auto_record_type_suggestion = relationship( - "AutoRecordTypeSuggestion", uselist=False, back_populates="url") + AnnotationAutoRecordType, uselist=False, back_populates="url") user_record_type_suggestions = relationship( - "UserRecordTypeSuggestion", back_populates="url") + AnnotationUserRecordType, back_populates="url") # Relvant/URL Type auto_relevant_suggestion = relationship( - "AutoRelevantSuggestion", uselist=False, back_populates="url") + AnnotationAutoURLType, uselist=False, back_populates="url") user_relevant_suggestions = relationship( - "UserURLTypeSuggestion", back_populates="url") + AnnotationUserURLType, back_populates="url") reviewing_user = relationship( "ReviewingUserURL", uselist=False, back_populates="url") diff --git a/src/db/models/impl/url/suggestion/anonymous/__init__.py b/src/db/models/impl/url/suggestion/anonymous/__init__.py deleted file mode 100644 index fddc715f..00000000 --- a/src/db/models/impl/url/suggestion/anonymous/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from src.db.models.impl.url.suggestion.anonymous.session.sqlalchemy import AnonymousSession \ No newline at end of file diff --git a/src/db/models/impl/url/suggestion/name/pydantic.py b/src/db/models/impl/url/suggestion/name/pydantic.py deleted file mode 100644 index 244e02c2..00000000 --- a/src/db/models/impl/url/suggestion/name/pydantic.py +++ /dev/null @@ -1,17 +0,0 @@ -from pydantic import Field - -from src.db.models.impl.url.suggestion.location.auto.subtask.constants import MAX_SUGGESTION_LENGTH -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion -from src.db.templates.markers.bulk.insert import BulkInsertableModel - - -class URLNameSuggestionPydantic(BulkInsertableModel): - - url_id: int - suggestion: str = Field(..., max_length=MAX_SUGGESTION_LENGTH) - source: NameSuggestionSource - - @classmethod - def sa_model(cls) -> type[URLNameSuggestion]: - return URLNameSuggestion \ No newline at end of file diff --git a/src/db/models/views/url_anno_count.py b/src/db/models/views/url_anno_count.py index 2e910afb..f3909b39 100644 --- a/src/db/models/views/url_anno_count.py +++ b/src/db/models/views/url_anno_count.py @@ -5,7 +5,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.auto_location_id_subtasks anno on u.id = anno.url_id + inner join public.annotation__auto__location__subtasks anno on u.id = anno.url_id group by u.id ) , auto_agency_count as ( @@ -13,7 +13,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.url_auto_agency_id_subtasks anno on u.id = anno.url_id + inner join public.annotation__auto__agency__subtasks anno on u.id = anno.url_id group by u.id ) , auto_url_type_count as ( @@ -21,7 +21,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.auto_relevant_suggestions anno on u.id = anno.url_id + inner join public.annotation__auto__url_type anno on u.id = anno.url_id group by u.id ) , auto_record_type_count as ( @@ -29,7 +29,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.auto_record_type_suggestions anno on u.id = anno.url_id + inner join public.annotation__auto__record_type anno on u.id = anno.url_id group by u.id ) , user_location_count as ( @@ -37,7 +37,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.user_location_suggestions anno on u.id = anno.url_id + inner join public.annotation__user__location anno on u.id = anno.url_id group by u.id ) , user_agency_count as ( @@ -45,7 +45,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.user_url_agency_suggestions anno on u.id = anno.url_id + inner join public.annotation__user__agency anno on u.id = anno.url_id group by u.id ) , user_url_type_count as ( @@ -53,7 +53,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.user_url_type_suggestions anno on u.id = anno.url_id + inner join public.annotation__user__url_type anno on u.id = anno.url_id group by u.id ) , user_record_type_count as ( @@ -61,7 +61,7 @@ u.id, count(anno.url_id) as cnt from urls u - inner join public.user_record_type_suggestions anno on u.id = anno.url_id + inner join public.annotation__user__record_type anno on u.id = anno.url_id group by u.id ) select diff --git a/src/db/models/views/url_annotations_flags.py b/src/db/models/views/url_annotations_flags.py index c133fbfc..b194a5e0 100644 --- a/src/db/models/views/url_annotations_flags.py +++ b/src/db/models/views/url_annotations_flags.py @@ -11,12 +11,12 @@ CASE WHEN cua.url_id IS NOT NULL THEN TRUE ELSE FALSE END AS has_confirmed_agency, CASE WHEN ruu.url_id IS NOT NULL THEN TRUE ELSE FALSE END AS was_reviewed FROM urls u - LEFT JOIN public.auto_record_type_suggestions arts ON u.id = arts.url_id - LEFT JOIN public.auto_relevant_suggestions ars ON u.id = ars.url_id + LEFT JOIN public.annotation__auto__record_type arts ON u.id = arts.url_id + LEFT JOIN public.annotation__auto__url_type ars ON u.id = ars.url_id LEFT JOIN public.{URL_AUTO_AGENCY_SUGGESTIONS_TABLE_NAME} auas ON u.id = auas.url_id - LEFT JOIN public.user_record_type_suggestions urts ON u.id = urts.url_id - LEFT JOIN public.user_relevant_suggestions urs ON u.id = urs.url_id - LEFT JOIN public.user_url_agency_suggestions uuas ON u.id = uuas.url_id + LEFT JOIN public.annotation__user__record_type urts ON u.id = urts.url_id + LEFT JOIN public.annotation__user__url_type urs ON u.id = urs.url_id + LEFT JOIN public.annotation__user__agency uuas ON u.id = uuas.url_id LEFT JOIN public.reviewing_user_url ruu ON u.id = ruu.url_id LEFT JOIN public.link_agencies__urls cua on u.id = cua.url_id ) diff --git a/src/db/queries/implementations/anonymous_session.py b/src/db/queries/implementations/anonymous_session.py index 0ff00ea3..a2fbf346 100644 --- a/src/db/queries/implementations/anonymous_session.py +++ b/src/db/queries/implementations/anonymous_session.py @@ -2,7 +2,7 @@ from sqlalchemy.ext.asyncio import AsyncSession -from src.db.models.impl.url.suggestion.anonymous import AnonymousSession +from src.db.models.impl.anon_session.sqlalchemy import AnonymousSession from src.db.queries.base.builder import QueryBuilderBase diff --git a/src/db/queries/implementations/core/common/annotation_exists_/constants.py b/src/db/queries/implementations/core/common/annotation_exists_/constants.py index 190291ef..5851b42b 100644 --- a/src/db/queries/implementations/core/common/annotation_exists_/constants.py +++ b/src/db/queries/implementations/core/common/annotation_exists_/constants.py @@ -1,15 +1,15 @@ -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.record_type.auto import AutoRecordTypeSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.auto.sqlalchemy import AutoRelevantSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.auto.sqlalchemy import AnnotationAutoURLType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType ALL_ANNOTATION_MODELS = [ - AutoRecordTypeSuggestion, - AutoRelevantSuggestion, - URLAutoAgencyIDSubtask, - UserURLTypeSuggestion, - UserRecordTypeSuggestion, - UserURLAgencySuggestion + AnnotationAutoRecordType, + AnnotationAutoURLType, + AnnotationAgencyAutoSubtask, + AnnotationUserURLType, + AnnotationUserRecordType, + AnnotationAgencyUser ] diff --git a/src/db/queries/implementations/core/metrics/urls/aggregated/pending.py b/src/db/queries/implementations/core/metrics/urls/aggregated/pending.py index d609e2b3..be8a76f9 100644 --- a/src/db/queries/implementations/core/metrics/urls/aggregated/pending.py +++ b/src/db/queries/implementations/core/metrics/urls/aggregated/pending.py @@ -5,10 +5,10 @@ from src.api.endpoints.metrics.dtos.get.urls.aggregated.pending import GetMetricsURLsAggregatedPendingResponseDTO from src.collectors.enums import URLStatus +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.models.mixins import URLDependentMixin from src.db.queries.base.builder import QueryBuilderBase from src.db.queries.implementations.core.common.annotation_exists_.core import AnnotationExistsCTEQueryBuilder @@ -17,15 +17,15 @@ class PendingAnnotationExistsCTEQueryBuilder(AnnotationExistsCTEQueryBuilder): @property def has_user_relevant_annotation(self): - return self.get_exists_for_model(UserURLTypeSuggestion) + return self.get_exists_for_model(AnnotationUserURLType) @property def has_user_record_type_annotation(self): - return self.get_exists_for_model(UserRecordTypeSuggestion) + return self.get_exists_for_model(AnnotationUserRecordType) @property def has_user_agency_annotation(self): - return self.get_exists_for_model(UserURLAgencySuggestion) + return self.get_exists_for_model(AnnotationAgencyUser) def get_exists_for_model(self, model: Type[URLDependentMixin]): return self.query.c[ diff --git a/src/db/types.py b/src/db/types.py index c224a36c..0ff28637 100644 --- a/src/db/types.py +++ b/src/db/types.py @@ -1,10 +1,10 @@ from typing import TypeVar -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.queries.base.labels import LabelsBase -UserSuggestionType = UserURLAgencySuggestion | UserURLTypeSuggestion | UserRecordTypeSuggestion +UserSuggestionType = AnnotationAgencyUser | AnnotationUserURLType | AnnotationUserRecordType LabelsType = TypeVar("LabelsType", bound=LabelsBase) \ No newline at end of file diff --git a/tests/automated/integration/api/annotate/all/test_happy_path.py b/tests/automated/integration/api/annotate/all/test_happy_path.py index 49d8bd97..faed2220 100644 --- a/tests/automated/integration/api/annotate/all/test_happy_path.py +++ b/tests/automated/integration/api/annotate/all/test_happy_path.py @@ -8,13 +8,13 @@ from src.api.endpoints.annotate.all.post.models.name import AnnotationPostNameInfo from src.api.endpoints.annotate.all.post.models.request import AllAnnotationPostInfo from src.core.enums import RecordType +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion from src.db.models.impl.flag.url_validated.enums import URLType -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from tests.helpers.data_creator.models.creation_info.us_state import USStateCreationInfo from tests.helpers.setup.final_review.core import setup_for_get_next_url_for_final_review @@ -106,19 +106,19 @@ async def test_annotate_all( # Check that all annotations are present in the database # Check URL Type Suggestions - all_relevance_suggestions: list[UserURLTypeSuggestion] = await adb_client.get_all(UserURLTypeSuggestion) + all_relevance_suggestions: list[AnnotationUserURLType] = await adb_client.get_all(AnnotationUserURLType) assert len(all_relevance_suggestions) == 4 suggested_types: set[URLType] = {sugg.type for sugg in all_relevance_suggestions} assert suggested_types == {URLType.DATA_SOURCE, URLType.NOT_RELEVANT} # Should be one agency - all_agency_suggestions = await adb_client.get_all(UserURLAgencySuggestion) + all_agency_suggestions = await adb_client.get_all(AnnotationAgencyUser) assert len(all_agency_suggestions) == 3 suggested_agency_ids: set[int] = {sugg.agency_id for sugg in all_agency_suggestions} assert agency_id in suggested_agency_ids # Should be one record type - all_record_type_suggestions = await adb_client.get_all(UserRecordTypeSuggestion) + all_record_type_suggestions = await adb_client.get_all(AnnotationUserRecordType) assert len(all_record_type_suggestions) == 3 suggested_record_types: set[RecordType] = { sugg.record_type for sugg in all_record_type_suggestions @@ -126,7 +126,7 @@ async def test_annotate_all( assert RecordType.ACCIDENT_REPORTS.value in suggested_record_types # Confirm 3 Location Suggestions, with two belonging to California and one to Pennsylvania - all_location_suggestions = await adb_client.get_all(UserLocationSuggestion) + all_location_suggestions = await adb_client.get_all(AnnotationLocationUser) assert len(all_location_suggestions) == 2 location_ids: list[int] = [location_suggestion.location_id for location_suggestion in all_location_suggestions] assert set(location_ids) == {california.location_id, pennsylvania.location_id} @@ -166,7 +166,7 @@ async def test_annotate_all( assert user_suggestion.user_count == 1 # Confirm 3 name suggestions - name_suggestions: list[URLNameSuggestion] = await adb_client.get_all(URLNameSuggestion) + name_suggestions: list[AnnotationNameSuggestion] = await adb_client.get_all(AnnotationNameSuggestion) assert len(name_suggestions) == 3 suggested_names: set[str] = {name_suggestion.suggestion for name_suggestion in name_suggestions} assert "New Name" in suggested_names diff --git a/tests/automated/integration/api/annotate/anonymous/test_core.py b/tests/automated/integration/api/annotate/anonymous/test_core.py index fa39d6f4..48fb4f4d 100644 --- a/tests/automated/integration/api/annotate/anonymous/test_core.py +++ b/tests/automated/integration/api/annotate/anonymous/test_core.py @@ -10,13 +10,13 @@ from src.api.endpoints.annotate.anonymous.get.response import GetNextURLForAnonymousAnnotationResponse from src.core.enums import RecordType from src.db.dtos.url.mapping_.simple import SimpleURLMapping +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType from src.db.models.impl.flag.url_validated.enums import URLType -from src.db.models.impl.link.anonymous_sessions__name_suggestion import LinkAnonymousSessionNameSuggestion -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.annotation.name.anon.sqlalchemy import AnnotationNameAnonEndorsement from src.db.models.mixins import URLDependentMixin from tests.automated.integration.api.annotate.anonymous.helper import get_next_url_for_anonymous_annotation, \ post_and_get_next_url_for_anonymous_annotation @@ -81,10 +81,10 @@ async def test_annotate_anonymous( assert post_response_1.next_annotation.url_info.url_id != get_response_1.next_annotation.url_info.url_id for model in [ - AnonymousAnnotationAgency, - AnonymousAnnotationLocation, - AnonymousAnnotationRecordType, - AnonymousAnnotationURLType + AnnotationAgencyAnon, + AnnotationLocationAnon, + AnnotationAnonRecordType, + AnnotationAnonURLType ]: instances: list[URLDependentMixin] = await ddc.adb_client.get_all(model) assert len(instances) == 1 @@ -92,13 +92,13 @@ async def test_annotate_anonymous( assert instance.url_id == get_response_1.next_annotation.url_info.url_id # Check for existence of name suggestion (2 were added by setup) - name_suggestions: list[URLNameSuggestion] = await ddc.adb_client.get_all(URLNameSuggestion) + name_suggestions: list[AnnotationNameSuggestion] = await ddc.adb_client.get_all(AnnotationNameSuggestion) assert len(name_suggestions) == 3 # Check for existence of link - link_instances: list[LinkAnonymousSessionNameSuggestion] = await ddc.adb_client.get_all(LinkAnonymousSessionNameSuggestion) + link_instances: list[AnnotationNameAnonEndorsement] = await ddc.adb_client.get_all(AnnotationNameAnonEndorsement) assert len(link_instances) == 1 - link_instance: LinkAnonymousSessionNameSuggestion = link_instances[0] + link_instance: AnnotationNameAnonEndorsement = link_instances[0] assert link_instance.session_id == session_id # Run again without giving session ID, confirm original URL returned diff --git a/tests/automated/integration/api/submit/data_source/test_core.py b/tests/automated/integration/api/submit/data_source/test_core.py index 558327c3..5a8fb103 100644 --- a/tests/automated/integration/api/submit/data_source/test_core.py +++ b/tests/automated/integration/api/submit/data_source/test_core.py @@ -7,6 +7,11 @@ from src.collectors.enums import URLStatus from src.core.enums import RecordType, BatchStatus from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType from src.db.models.impl.batch.sqlalchemy import Batch from src.db.models.impl.flag.url_validated.enums import URLType from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL @@ -15,11 +20,6 @@ from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \ RetentionScheduleEnum, AccessTypeEnum from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion from tests.helpers.api_test_helper import APITestHelper from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo @@ -96,31 +96,31 @@ async def test_submit_data_source( assert batch_url_link.url_id == url.id # Check for anonymous annotations - url_type_suggestion: AnonymousAnnotationURLType = await adb_client.one_or_none_model(AnonymousAnnotationURLType) + url_type_suggestion: AnnotationAnonURLType = await adb_client.one_or_none_model(AnnotationAnonURLType) assert url_type_suggestion is not None assert url_type_suggestion.url_id == url.id assert url_type_suggestion.url_type == URLType.DATA_SOURCE session_id: UUID = url_type_suggestion.session_id # Check for Location Suggestion - location_suggestion: AnonymousAnnotationLocation = await adb_client.one_or_none_model(AnonymousAnnotationLocation) + location_suggestion: AnnotationLocationAnon = await adb_client.one_or_none_model(AnnotationLocationAnon) assert location_suggestion is not None assert location_suggestion.location_id == pittsburgh_locality.location_id assert location_suggestion.session_id == session_id # Check for Agency Suggestion - agency_suggestion: AnonymousAnnotationAgency = await adb_client.one_or_none_model(AnonymousAnnotationAgency) + agency_suggestion: AnnotationAgencyAnon = await adb_client.one_or_none_model(AnnotationAgencyAnon) assert agency_suggestion is not None assert agency_suggestion.agency_id == test_agency_id assert agency_suggestion.session_id == session_id # Check for Name Suggestion - name_suggestion: URLNameSuggestion = await adb_client.one_or_none_model(URLNameSuggestion) + name_suggestion: AnnotationNameSuggestion = await adb_client.one_or_none_model(AnnotationNameSuggestion) assert name_suggestion is not None assert name_suggestion.suggestion == "Example name" # Check for Record Type Suggestion - record_type_suggestion: AnonymousAnnotationRecordType = await adb_client.one_or_none_model(AnonymousAnnotationRecordType) + record_type_suggestion: AnnotationAnonRecordType = await adb_client.one_or_none_model(AnnotationAnonRecordType) assert record_type_suggestion.record_type == RecordType.COMPLAINTS_AND_MISCONDUCT assert record_type_suggestion.session_id == session_id diff --git a/tests/automated/integration/api/submit/test_url_maximal.py b/tests/automated/integration/api/submit/test_url_maximal.py index e57770fb..27fbfe2f 100644 --- a/tests/automated/integration/api/submit/test_url_maximal.py +++ b/tests/automated/integration/api/submit/test_url_maximal.py @@ -5,14 +5,14 @@ from src.api.endpoints.submit.url.models.response import URLSubmissionResponse from src.core.enums import RecordType from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType from tests.helpers.api_test_helper import APITestHelper from tests.helpers.data_creator.core import DBDataCreator from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo @@ -56,21 +56,21 @@ async def test_maximal( link: LinkUserSubmittedURL = links[0] assert link.url_id == url_id - agen_suggs: list[UserURLAgencySuggestion] = await adb_client.get_all(UserURLAgencySuggestion) + agen_suggs: list[AnnotationAgencyUser] = await adb_client.get_all(AnnotationAgencyUser) assert len(agen_suggs) == 1 - agen_sugg: UserURLAgencySuggestion = agen_suggs[0] + agen_sugg: AnnotationAgencyUser = agen_suggs[0] assert agen_sugg.url_id == url_id assert agen_sugg.agency_id == agency_id - loc_suggs: list[UserLocationSuggestion] = await adb_client.get_all(UserLocationSuggestion) + loc_suggs: list[AnnotationLocationUser] = await adb_client.get_all(AnnotationLocationUser) assert len(loc_suggs) == 1 - loc_sugg: UserLocationSuggestion = loc_suggs[0] + loc_sugg: AnnotationLocationUser = loc_suggs[0] assert loc_sugg.url_id == url_id assert loc_sugg.location_id == pittsburgh_locality.location_id - name_sugg: list[URLNameSuggestion] = await adb_client.get_all(URLNameSuggestion) + name_sugg: list[AnnotationNameSuggestion] = await adb_client.get_all(AnnotationNameSuggestion) assert len(name_sugg) == 1 - name_sugg: URLNameSuggestion = name_sugg[0] + name_sugg: AnnotationNameSuggestion = name_sugg[0] assert name_sugg.url_id == url_id assert name_sugg.suggestion == "Example URL" assert name_sugg.source == NameSuggestionSource.USER @@ -80,8 +80,8 @@ async def test_maximal( name_link_sugg: LinkUserNameSuggestion = name_link_suggs[0] assert name_link_sugg.suggestion_id == name_sugg.id - rec_suggs: list[UserRecordTypeSuggestion] = await adb_client.get_all(UserRecordTypeSuggestion) + rec_suggs: list[AnnotationUserRecordType] = await adb_client.get_all(AnnotationUserRecordType) assert len(rec_suggs) == 1 - rec_sugg: UserRecordTypeSuggestion = rec_suggs[0] + rec_sugg: AnnotationUserRecordType = rec_suggs[0] assert rec_sugg.url_id == url_id assert rec_sugg.record_type == RecordType.INCARCERATION_RECORDS.value diff --git a/tests/automated/integration/api/url/by_id/delete/test_any_url.py b/tests/automated/integration/api/url/by_id/delete/test_any_url.py index 50b3ca0c..49f7a407 100644 --- a/tests/automated/integration/api/url/by_id/delete/test_any_url.py +++ b/tests/automated/integration/api/url/by_id/delete/test_any_url.py @@ -7,6 +7,20 @@ from src.db.client.async_ import AsyncDatabaseClient from src.db.dtos.url.mapping_.simple import SimpleURLMapping from src.db.enums import ChangeLogOperationType +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType from src.db.models.impl.change_log import ChangeLog from src.db.models.impl.flag.checked_for_ia.sqlalchemy import FlagURLCheckedForInternetArchives from src.db.models.impl.flag.root_url.sqlalchemy import FlagRootURL @@ -15,7 +29,7 @@ from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL from src.db.models.impl.link.url_redirect_url.sqlalchemy import LinkURLRedirectURL from src.db.models.impl.link.urls_root_url.sqlalchemy import LinkURLRootURL -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.models.impl.link.user_suggestion_not_found.agency.sqlalchemy import LinkUserSuggestionAgencyNotFound from src.db.models.impl.link.user_suggestion_not_found.location.sqlalchemy import LinkUserSuggestionLocationNotFound from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL @@ -26,24 +40,10 @@ from src.db.models.impl.url.internet_archives.probe.sqlalchemy import URLInternetArchivesProbeMetadata from src.db.models.impl.url.internet_archives.save.sqlalchemy import URLInternetArchivesSaveMetadata from src.db.models.impl.url.screenshot.sqlalchemy import URLScreenshot -from src.db.models.impl.url.suggestion.agency.subtask.enum import SubtaskDetailCode, AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion -from src.db.models.impl.url.suggestion.record_type.auto import AutoRecordTypeSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.auto.sqlalchemy import AutoRelevantSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.auto.sqlalchemy import AnnotationAutoURLType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType from src.db.models.impl.url.task_error.sqlalchemy import URLTaskError from src.db.models.impl.url.web_metadata.sqlalchemy import URLWebMetadata from src.db.queries.implementations.anonymous_session import MakeAnonymousSessionQueryBuilder @@ -128,34 +128,34 @@ async def _check_results( # ANNOTATIONS ## AUTO ### Agency - URLAutoAgencyIDSubtask, - AgencyIDSubtaskSuggestion, + AnnotationAgencyAutoSubtask, + AnnotationAgencyAutoSuggestion, ### Record Type - AutoRecordTypeSuggestion, + AnnotationAutoRecordType, ### URL Type - AutoRelevantSuggestion, + AnnotationAutoURLType, ### Location - AutoLocationIDSubtask, - LocationIDSubtaskSuggestion, + AnnotationLocationAutoSubtask, + AnnotationLocationAutoSuggestion, ## USER ### Agency - UserURLAgencySuggestion, + AnnotationAgencyUser, ### Record Type - UserRecordTypeSuggestion, + AnnotationUserRecordType, ### URL Type - UserURLTypeSuggestion, + AnnotationUserURLType, ### Location - UserLocationSuggestion, - URLNameSuggestion, + AnnotationLocationUser, + AnnotationNameSuggestion, ## ANONYMOUS ### Agency - AnonymousAnnotationAgency, + AnnotationAgencyAnon, ### Location - AnonymousAnnotationLocation, + AnnotationLocationAnon, ### Record Type - AnonymousAnnotationRecordType, + AnnotationAnonRecordType, ### URL Type - AnonymousAnnotationURLType, + AnnotationAnonURLType, ] for model in models: assert await dbc.get_all(model) == [] @@ -316,7 +316,7 @@ async def _setup( ### Agency #### Subtask agency_subtask_id: int = await dbc.add( - URLAutoAgencyIDSubtask( + AnnotationAgencyAutoSubtask( url_id=url.url_id, task_id=task_id, agencies_found=True, @@ -327,7 +327,7 @@ async def _setup( ) ### Suggestion await dbc.add( - AgencyIDSubtaskSuggestion( + AnnotationAgencyAutoSuggestion( subtask_id=agency_subtask_id, agency_id=agency_id, confidence=60 @@ -335,14 +335,14 @@ async def _setup( ) ### Record Type await dbc.add( - AutoRecordTypeSuggestion( + AnnotationAutoRecordType( url_id=url.url_id, record_type=RecordType.BOOKING_REPORTS.value ) ) ### Relevant await dbc.add( - AutoRelevantSuggestion( + AnnotationAutoURLType( url_id=url.url_id, relevant=True, confidence=0.5, @@ -352,7 +352,7 @@ async def _setup( ### Location #### Subtask location_subtask_id: int = await dbc.add( - AutoLocationIDSubtask( + AnnotationLocationAutoSubtask( url_id=url.url_id, task_id=task_id, locations_found=True, @@ -362,7 +362,7 @@ async def _setup( ) #### Suggestion await dbc.add( - LocationIDSubtaskSuggestion( + AnnotationLocationAutoSuggestion( subtask_id=location_subtask_id, location_id=pittsburgh_id, confidence=50 @@ -371,7 +371,7 @@ async def _setup( ## USER ### Agency await dbc.add( - UserURLAgencySuggestion( + AnnotationAgencyUser( url_id=url.url_id, user_id=1, agency_id=agency_id, @@ -380,7 +380,7 @@ async def _setup( ) ### Record Type await dbc.add( - UserRecordTypeSuggestion( + AnnotationUserRecordType( url_id=url.url_id, user_id=1, record_type=RecordType.BOOKING_REPORTS.value, @@ -388,7 +388,7 @@ async def _setup( ) ### URL Type await dbc.add( - UserURLTypeSuggestion( + AnnotationUserURLType( url_id=url.url_id, type=URLType.INDIVIDUAL_RECORD, user_id=1 @@ -396,7 +396,7 @@ async def _setup( ) ### Location await dbc.add( - UserLocationSuggestion( + AnnotationLocationUser( url_id=url.url_id, location_id=pittsburgh_id, user_id=1, @@ -404,7 +404,7 @@ async def _setup( ) ### Name name_suggestion_id: int = await dbc.add( - URLNameSuggestion( + AnnotationNameSuggestion( url_id=url.url_id, suggestion="Test Name", source=NameSuggestionSource.USER, @@ -423,25 +423,25 @@ async def _setup( ## ANONYMOUS for model in [ ### Agency - AnonymousAnnotationAgency( + AnnotationAgencyAnon( url_id=url.url_id, agency_id=agency_id, session_id=session_id, ), ### Record Type - AnonymousAnnotationRecordType( + AnnotationAnonRecordType( url_id=url.url_id, record_type=RecordType.BOOKING_REPORTS.value, session_id=session_id, ), ### URL Type - AnonymousAnnotationURLType( + AnnotationAnonURLType( url_id=url.url_id, url_type=URLType.INDIVIDUAL_RECORD, session_id=session_id, ), ### Location - AnonymousAnnotationLocation( + AnnotationLocationAnon( url_id=url.url_id, location_id=pittsburgh_id, session_id=session_id diff --git a/tests/automated/integration/readonly/setup/annotations.py b/tests/automated/integration/readonly/setup/annotations.py index b07bbd9f..ab2ef13e 100644 --- a/tests/automated/integration/readonly/setup/annotations.py +++ b/tests/automated/integration/readonly/setup/annotations.py @@ -1,13 +1,13 @@ from src.core.enums import RecordType from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion from src.db.models.impl.flag.url_validated.enums import URLType -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion -from src.db.models.impl.url.suggestion.agency.user import UserURLAgencySuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion -from src.db.models.impl.url.suggestion.record_type.user import UserRecordTypeSuggestion -from src.db.models.impl.url.suggestion.url_type.user import UserURLTypeSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.record_type.user.user import AnnotationUserRecordType +from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationUserURLType async def add_full_data_sources_annotations( @@ -17,7 +17,7 @@ async def add_full_data_sources_annotations( location_id: int, adb_client: AsyncDatabaseClient ) -> None: - name_suggestion = URLNameSuggestion( + name_suggestion = AnnotationNameSuggestion( url_id=url_id, suggestion="Name suggestion", source=NameSuggestionSource.USER @@ -26,12 +26,12 @@ async def add_full_data_sources_annotations( name_suggestion, return_id=True ) - url_type_suggestion = UserURLTypeSuggestion( + url_type_suggestion = AnnotationUserURLType( url_id=url_id, user_id=user_id, type=URLType.DATA_SOURCE ) - record_type_suggestion = UserRecordTypeSuggestion( + record_type_suggestion = AnnotationUserRecordType( user_id=user_id, url_id=url_id, record_type=RecordType.RECORDS_REQUEST_INFO.value @@ -40,12 +40,12 @@ async def add_full_data_sources_annotations( user_id=user_id, suggestion_id=name_suggestion_id, ) - agency_suggestion = UserURLAgencySuggestion( + agency_suggestion = AnnotationAgencyUser( agency_id=agency_id, url_id=url_id, user_id=user_id, ) - location_suggestion = UserLocationSuggestion( + location_suggestion = AnnotationLocationUser( location_id=location_id, url_id=url_id, user_id=user_id, @@ -64,7 +64,7 @@ async def add_minimal_not_relevant_annotation( user_id: int, adb_client: AsyncDatabaseClient ) -> None: - url_type_suggestion = UserURLTypeSuggestion( + url_type_suggestion = AnnotationUserURLType( url_id=url_id, user_id=user_id, type=URLType.NOT_RELEVANT diff --git a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/batch_link/test_core.py b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/batch_link/test_core.py index b39d74ca..e838ee3e 100644 --- a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/batch_link/test_core.py +++ b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/batch_link/test_core.py @@ -2,10 +2,10 @@ from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from src.db.models.impl.link.agency_batch.sqlalchemy import LinkAgencyBatch -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion from tests.helpers.batch_creation_parameters.core import TestBatchCreationParameters from tests.helpers.batch_creation_parameters.url_creation_parameters import TestURLCreationParameters from tests.helpers.data_creator.core import DBDataCreator @@ -49,14 +49,14 @@ async def test_batch_link_subtask( assert not await operator.meets_task_prerequisites() assert operator._subtask is None - subtasks: list[URLAutoAgencyIDSubtask] = await adb_client.get_all(URLAutoAgencyIDSubtask) + subtasks: list[AnnotationAgencyAutoSubtask] = await adb_client.get_all(AnnotationAgencyAutoSubtask) assert len(subtasks) == 2 - subtask: URLAutoAgencyIDSubtask = subtasks[0] + subtask: AnnotationAgencyAutoSubtask = subtasks[0] assert subtask.type == AutoAgencyIDSubtaskType.BATCH_LINK assert subtask.agencies_found - suggestions: list[AgencyIDSubtaskSuggestion] = await adb_client.get_all(AgencyIDSubtaskSuggestion) + suggestions: list[AnnotationAgencyAutoSuggestion] = await adb_client.get_all(AnnotationAgencyAutoSuggestion) assert len(suggestions) == 2 assert all(sugg.confidence == 80 for sugg in suggestions) diff --git a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/ckan/test_core.py b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/ckan/test_core.py index 4ec99967..a1ba703f 100644 --- a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/ckan/test_core.py +++ b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/ckan/test_core.py @@ -4,9 +4,9 @@ from src.core.tasks.base.run_info import TaskOperatorRunInfo from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from tests.helpers.asserts import assert_task_run_success from tests.helpers.data_creator.core import DBDataCreator @@ -57,15 +57,15 @@ async def test_ckan_subtask( assert operator._subtask is None # Verify results - subtasks: list[URLAutoAgencyIDSubtask] = await adb_client.get_all(URLAutoAgencyIDSubtask) + subtasks: list[AnnotationAgencyAutoSubtask] = await adb_client.get_all(AnnotationAgencyAutoSubtask) assert len(subtasks) == 1 - subtask: URLAutoAgencyIDSubtask = subtasks[0] + subtask: AnnotationAgencyAutoSubtask = subtasks[0] assert subtask.type == AutoAgencyIDSubtaskType.CKAN assert subtask.url_id == applicable_url_id subtask_id: int = subtask.id - suggestions: list[AgencyIDSubtaskSuggestion] = await adb_client.get_all( - AgencyIDSubtaskSuggestion + suggestions: list[AnnotationAgencyAutoSuggestion] = await adb_client.get_all( + AnnotationAgencyAutoSuggestion ) assert len(suggestions) == 2 assert {suggestion.agency_id for suggestion in suggestions} == { diff --git a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/homepage_match/test_happy_path.py b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/homepage_match/test_happy_path.py index 7575f37e..7e72b733 100644 --- a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/homepage_match/test_happy_path.py +++ b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/homepage_match/test_happy_path.py @@ -6,10 +6,10 @@ from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator from src.db.client.async_ import AsyncDatabaseClient from src.db.dtos.url.mapping_.simple import SimpleURLMapping +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from src.db.models.impl.flag.url_validated.enums import URLType -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType, SubtaskDetailCode -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion from tests.automated.integration.tasks.url.impl.asserts import assert_task_ran_without_error from tests.helpers.data_creator.core import DBDataCreator @@ -113,17 +113,17 @@ async def test_homepage_match( adb_client: AsyncDatabaseClient = db_data_creator.adb_client # Confirm presence of subtasks - subtasks: list[URLAutoAgencyIDSubtask] = await adb_client.get_all(URLAutoAgencyIDSubtask) + subtasks: list[AnnotationAgencyAutoSubtask] = await adb_client.get_all(AnnotationAgencyAutoSubtask) assert len(subtasks) == 2 # Confirm both listed as agencies found assert all(subtask.agencies_found for subtask in subtasks) - url_id_to_subtask: dict[int, URLAutoAgencyIDSubtask] = { + url_id_to_subtask: dict[int, AnnotationAgencyAutoSubtask] = { subtask.url_id: subtask for subtask in subtasks } - single_subtask: URLAutoAgencyIDSubtask = url_id_to_subtask[single_url_id] - multi_subtask: URLAutoAgencyIDSubtask = url_id_to_subtask[multi_url_id] + single_subtask: AnnotationAgencyAutoSubtask = url_id_to_subtask[single_url_id] + multi_subtask: AnnotationAgencyAutoSubtask = url_id_to_subtask[multi_url_id] # Check subtasks have expected detail codes assert single_subtask.detail == SubtaskDetailCode.HOMEPAGE_SINGLE_AGENCY @@ -131,16 +131,16 @@ async def test_homepage_match( # Get suggestions - suggestions: list[AgencyIDSubtaskSuggestion] = await adb_client.get_all(AgencyIDSubtaskSuggestion) + suggestions: list[AnnotationAgencyAutoSuggestion] = await adb_client.get_all(AnnotationAgencyAutoSuggestion) assert len(suggestions) == 3 # Confirm each suggestion properly linked to expected subtask - subtask_id_to_suggestions: dict[int, list[AgencyIDSubtaskSuggestion]] = defaultdict(list) + subtask_id_to_suggestions: dict[int, list[AnnotationAgencyAutoSuggestion]] = defaultdict(list) for suggestion in suggestions: subtask_id_to_suggestions[suggestion.subtask_id].append(suggestion) # Check Single Agency Case Suggestion - single_suggestion: AgencyIDSubtaskSuggestion = \ + single_suggestion: AnnotationAgencyAutoSuggestion = \ subtask_id_to_suggestions[single_subtask.id][0] # Check Single Agency Case Suggestion has expected agency assert single_suggestion.agency_id == single_agency_id @@ -148,7 +148,7 @@ async def test_homepage_match( assert single_suggestion.confidence == 95 # Check Multi Agency Case Suggestion - multi_suggestions: list[AgencyIDSubtaskSuggestion] = subtask_id_to_suggestions[multi_subtask.id] + multi_suggestions: list[AnnotationAgencyAutoSuggestion] = subtask_id_to_suggestions[multi_subtask.id] # Check Multi Agency Case Suggestion has expected agencies assert {suggestion.agency_id for suggestion in multi_suggestions} \ == set(multi_agency_ids) diff --git a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/muckrock/test_core.py b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/muckrock/test_core.py index af41354d..aa38b33b 100644 --- a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/muckrock/test_core.py +++ b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/muckrock/test_core.py @@ -6,9 +6,9 @@ from src.core.tasks.base.run_info import TaskOperatorRunInfo from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from tests.helpers.asserts import assert_task_run_success from tests.helpers.data_creator.core import DBDataCreator @@ -93,15 +93,15 @@ async def test_muckrock_subtask( assert operator._subtask is None # Verify results - subtasks: list[URLAutoAgencyIDSubtask] = await adb_client.get_all(URLAutoAgencyIDSubtask) + subtasks: list[AnnotationAgencyAutoSubtask] = await adb_client.get_all(AnnotationAgencyAutoSubtask) assert len(subtasks) == 1 - subtask: URLAutoAgencyIDSubtask = subtasks[0] + subtask: AnnotationAgencyAutoSubtask = subtasks[0] assert subtask.type == AutoAgencyIDSubtaskType.MUCKROCK assert subtask.url_id == applicable_url_id subtask_id: int = subtask.id - suggestions: list[AgencyIDSubtaskSuggestion] = await adb_client.get_all( - AgencyIDSubtaskSuggestion + suggestions: list[AnnotationAgencyAutoSuggestion] = await adb_client.get_all( + AnnotationAgencyAutoSuggestion ) assert len(suggestions) == 2 assert {suggestion.agency_id for suggestion in suggestions} == { diff --git a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_multi_agency_location.py b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_multi_agency_location.py index 3da841a1..0df07b79 100644 --- a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_multi_agency_location.py +++ b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_multi_agency_location.py @@ -2,9 +2,9 @@ from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from tests.helpers.data_creator.core import DBDataCreator from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo from tests.helpers.run import run_task_and_confirm_success @@ -53,16 +53,16 @@ async def test_multi_agency_location( assert not await operator.meets_task_prerequisites() # Check for presence of subtask - subtasks: list[URLAutoAgencyIDSubtask] = await adb_client.get_all(URLAutoAgencyIDSubtask) + subtasks: list[AnnotationAgencyAutoSubtask] = await adb_client.get_all(AnnotationAgencyAutoSubtask) assert len(subtasks) == 1 - subtask: URLAutoAgencyIDSubtask = subtasks[0] + subtask: AnnotationAgencyAutoSubtask = subtasks[0] assert subtask.type == AutoAgencyIDSubtaskType.NLP_LOCATION_MATCH # Confirm subtask lists agencies found assert subtask.agencies_found # Confirm multiple agency suggestions in database - suggestions: list[AgencyIDSubtaskSuggestion] = await adb_client.get_all(AgencyIDSubtaskSuggestion) + suggestions: list[AnnotationAgencyAutoSuggestion] = await adb_client.get_all(AnnotationAgencyAutoSuggestion) assert len(suggestions) == 2 # Confirm confidence of location suggestion is distributed evenly among agency suggestions diff --git a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_single_agency_location.py b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_single_agency_location.py index ecec3071..6e1ef42d 100644 --- a/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_single_agency_location.py +++ b/tests/automated/integration/tasks/url/impl/agency_identification/subtasks/nlp_location_match/end_to_end/test_single_agency_location.py @@ -2,9 +2,9 @@ from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator from src.db.client.async_ import AsyncDatabaseClient -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.sqlalchemy import URLAutoAgencyIDSubtask -from src.db.models.impl.url.suggestion.agency.suggestion.sqlalchemy import AgencyIDSubtaskSuggestion +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.sqlalchemy import AnnotationAgencyAutoSubtask +from src.db.models.impl.annotation.agency.auto.suggestion.sqlalchemy import AnnotationAgencyAutoSuggestion from tests.helpers.data_creator.core import DBDataCreator from tests.helpers.data_creator.models.creation_info.county import CountyCreationInfo from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo @@ -59,18 +59,18 @@ async def test_single_agency_location( assert not await operator.meets_task_prerequisites() # Check for presence of subtask - subtasks: list[URLAutoAgencyIDSubtask] = await adb_client.get_all(URLAutoAgencyIDSubtask) + subtasks: list[AnnotationAgencyAutoSubtask] = await adb_client.get_all(AnnotationAgencyAutoSubtask) assert len(subtasks) == 1 - subtask: URLAutoAgencyIDSubtask = subtasks[0] + subtask: AnnotationAgencyAutoSubtask = subtasks[0] assert subtask.type == AutoAgencyIDSubtaskType.NLP_LOCATION_MATCH # Confirm subtask lists agencies found assert subtask.agencies_found # Confirm single agency suggestion in database - suggestions: list[AgencyIDSubtaskSuggestion] = await adb_client.get_all(AgencyIDSubtaskSuggestion) + suggestions: list[AnnotationAgencyAutoSuggestion] = await adb_client.get_all(AnnotationAgencyAutoSuggestion) assert len(suggestions) == 1 # Confirm confidence of agency suggestion equal to location suggestion - suggestion: AgencyIDSubtaskSuggestion = suggestions[0] + suggestion: AnnotationAgencyAutoSuggestion = suggestions[0] assert suggestion.confidence == 68 diff --git a/tests/automated/integration/tasks/url/impl/agency_identification/survey/test_survey_flag.py b/tests/automated/integration/tasks/url/impl/agency_identification/survey/test_survey_flag.py index 8ace042e..74e31306 100644 --- a/tests/automated/integration/tasks/url/impl/agency_identification/survey/test_survey_flag.py +++ b/tests/automated/integration/tasks/url/impl/agency_identification/survey/test_survey_flag.py @@ -2,7 +2,7 @@ from src.collectors.enums import CollectorType from src.core.tasks.url.operators.agency_identification.core import AgencyIdentificationTaskOperator -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType from tests.helpers.data_creator.core import DBDataCreator @pytest.mark.asyncio diff --git a/tests/automated/integration/tasks/url/impl/auto_name/test_core.py b/tests/automated/integration/tasks/url/impl/auto_name/test_core.py index c0500d99..66c09017 100644 --- a/tests/automated/integration/tasks/url/impl/auto_name/test_core.py +++ b/tests/automated/integration/tasks/url/impl/auto_name/test_core.py @@ -1,8 +1,8 @@ import pytest from src.core.tasks.url.operators.auto_name.core import AutoNameURLTaskOperator -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion from tests.helpers.data_creator.core import DBDataCreator from tests.helpers.run import run_task_and_confirm_success @@ -31,9 +31,9 @@ async def test_core( assert not await operator.meets_task_prerequisites() # Confirm suggestion was added - suggestions: list[URLNameSuggestion] = await db_data_creator.adb_client.get_all(URLNameSuggestion) + suggestions: list[AnnotationNameSuggestion] = await db_data_creator.adb_client.get_all(AnnotationNameSuggestion) assert len(suggestions) == 1 - suggestion: URLNameSuggestion = suggestions[0] + suggestion: AnnotationNameSuggestion = suggestions[0] assert suggestion.url_id == url_id assert suggestion.suggestion == "test html content" assert suggestion.source == NameSuggestionSource.HTML_METADATA_TITLE \ No newline at end of file diff --git a/tests/automated/integration/tasks/url/impl/auto_relevant/test_task.py b/tests/automated/integration/tasks/url/impl/auto_relevant/test_task.py index 3f4873f4..c9236f6c 100644 --- a/tests/automated/integration/tasks/url/impl/auto_relevant/test_task.py +++ b/tests/automated/integration/tasks/url/impl/auto_relevant/test_task.py @@ -1,11 +1,7 @@ -from collections import Counter - import pytest -from src.collectors.enums import URLStatus from src.core.tasks.url.operators.auto_relevant.core import URLAutoRelevantTaskOperator -from src.db.models.impl.url.core.sqlalchemy import URL -from src.db.models.impl.url.suggestion.url_type.auto.sqlalchemy import AutoRelevantSuggestion +from src.db.models.impl.annotation.url_type.auto.sqlalchemy import AnnotationAutoURLType from src.db.models.impl.url.task_error.sqlalchemy import URLTaskError from tests.automated.integration.tasks.url.impl.asserts import assert_prereqs_not_met, assert_prereqs_met from tests.automated.integration.tasks.url.impl.auto_relevant.setup import setup_operator, setup_urls @@ -31,7 +27,7 @@ async def test_url_auto_relevant_task(db_data_creator: DBDataCreator): adb_client = db_data_creator.adb_client # Confirm two annotations were created - suggestions: list[AutoRelevantSuggestion] = await adb_client.get_all(AutoRelevantSuggestion) + suggestions: list[AnnotationAutoURLType] = await adb_client.get_all(AnnotationAutoURLType) assert len(suggestions) == 2 for suggestion in suggestions: assert suggestion.url_id in url_ids diff --git a/tests/automated/integration/tasks/url/impl/location_identification/subtasks/batch_link/test_core.py b/tests/automated/integration/tasks/url/impl/location_identification/subtasks/batch_link/test_core.py index ab505627..0c5238ae 100644 --- a/tests/automated/integration/tasks/url/impl/location_identification/subtasks/batch_link/test_core.py +++ b/tests/automated/integration/tasks/url/impl/location_identification/subtasks/batch_link/test_core.py @@ -2,10 +2,10 @@ from src.core.tasks.url.operators.location_id.core import LocationIdentificationTaskOperator from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion from src.db.models.impl.link.location_batch.sqlalchemy import LinkLocationBatch -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion from tests.helpers.batch_creation_parameters.core import TestBatchCreationParameters from tests.helpers.batch_creation_parameters.url_creation_parameters import TestURLCreationParameters from tests.helpers.data_creator.core import DBDataCreator @@ -51,13 +51,13 @@ async def test_batch_link_subtask( assert not await operator.meets_task_prerequisites() assert operator._subtask is None - subtasks: list[AutoLocationIDSubtask] = await adb_client.get_all(AutoLocationIDSubtask) + subtasks: list[AnnotationLocationAutoSubtask] = await adb_client.get_all(AnnotationLocationAutoSubtask) assert len(subtasks) == 2 - subtask: AutoLocationIDSubtask = subtasks[0] + subtask: AnnotationLocationAutoSubtask = subtasks[0] assert subtask.type == LocationIDSubtaskType.BATCH_LINK assert subtask.locations_found - suggestions: list[LocationIDSubtaskSuggestion] = await adb_client.get_all(LocationIDSubtaskSuggestion) + suggestions: list[AnnotationLocationAutoSuggestion] = await adb_client.get_all(AnnotationLocationAutoSuggestion) assert len(suggestions) == 2 assert all(sugg.confidence == 80 for sugg in suggestions) diff --git a/tests/automated/integration/tasks/url/impl/location_identification/subtasks/nlp_location_frequency/end_to_end/test_core.py b/tests/automated/integration/tasks/url/impl/location_identification/subtasks/nlp_location_frequency/end_to_end/test_core.py index f8f0c821..8d8bd7c6 100644 --- a/tests/automated/integration/tasks/url/impl/location_identification/subtasks/nlp_location_frequency/end_to_end/test_core.py +++ b/tests/automated/integration/tasks/url/impl/location_identification/subtasks/nlp_location_frequency/end_to_end/test_core.py @@ -9,11 +9,11 @@ from src.core.tasks.url.operators.location_id.subtasks.models.subtask import AutoLocationIDSubtaskData from src.core.tasks.url.operators.location_id.subtasks.models.suggestion import LocationSuggestion from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion from src.db.models.impl.link.task_url import LinkTaskURL -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.pydantic import AutoLocationIDSubtaskPydantic -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion from src.db.models.impl.url.task_error.sqlalchemy import URLTaskError from tests.helpers.asserts import assert_task_run_success from tests.helpers.data_creator.core import DBDataCreator @@ -90,7 +90,7 @@ async def mock_process_inputs( assert {task_link.task_id for task_link in task_links} == {operator._task_id} # Confirm two subtasks were created - subtasks: list[AutoLocationIDSubtask] = await adb_client.get_all(AutoLocationIDSubtask) + subtasks: list[AnnotationLocationAutoSubtask] = await adb_client.get_all(AnnotationLocationAutoSubtask) assert len(subtasks) == 2 assert {subtask.url_id for subtask in subtasks} == set(url_ids) assert {subtask.task_id for subtask in subtasks} == {operator._task_id} @@ -108,7 +108,7 @@ async def mock_process_inputs( assert error_infos[0].error == "Test error" # Confirm two suggestions for happy path URL id - suggestions: list[LocationIDSubtaskSuggestion] = await adb_client.get_all(LocationIDSubtaskSuggestion) + suggestions: list[AnnotationLocationAutoSuggestion] = await adb_client.get_all(AnnotationLocationAutoSuggestion) assert len(suggestions) == 2 # Confirm expected agency ids assert {suggestion.location_id for suggestion in suggestions} == { diff --git a/tests/automated/integration/tasks/url/impl/location_identification/survey/test_survey_flag.py b/tests/automated/integration/tasks/url/impl/location_identification/survey/test_survey_flag.py index 338c604b..a5dca740 100644 --- a/tests/automated/integration/tasks/url/impl/location_identification/survey/test_survey_flag.py +++ b/tests/automated/integration/tasks/url/impl/location_identification/survey/test_survey_flag.py @@ -1,7 +1,7 @@ import pytest from src.core.tasks.url.operators.location_id.core import LocationIdentificationTaskOperator -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType from tests.helpers.data_creator.core import DBDataCreator diff --git a/tests/automated/integration/tasks/url/impl/test_url_record_type_task.py b/tests/automated/integration/tasks/url/impl/test_url_record_type_task.py index 57f41ded..d9f1de4f 100644 --- a/tests/automated/integration/tasks/url/impl/test_url_record_type_task.py +++ b/tests/automated/integration/tasks/url/impl/test_url_record_type_task.py @@ -3,7 +3,7 @@ import pytest from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.record_type.auto import AutoRecordTypeSuggestion +from src.db.models.impl.annotation.record_type.auto.sqlalchemy import AnnotationAutoRecordType from src.core.tasks.url.enums import TaskOperatorOutcome from src.core.tasks.url.operators.record_type.core import URLRecordTypeTaskOperator from src.core.enums import RecordType @@ -49,7 +49,7 @@ async def test_url_record_type_task(db_data_creator: DBDataCreator): assert task.url_error_count == 1 # Get metadata - suggestions = await db_data_creator.adb_client.get_all(AutoRecordTypeSuggestion) + suggestions = await db_data_creator.adb_client.get_all(AnnotationAutoRecordType) for suggestion in suggestions: assert suggestion.record_type == RecordType.ACCIDENT_REPORTS.value diff --git a/tests/automated/integration/tasks/url/impl/validate/helper.py b/tests/automated/integration/tasks/url/impl/validate/helper.py index 091fe5fa..ec9901dd 100644 --- a/tests/automated/integration/tasks/url/impl/validate/helper.py +++ b/tests/automated/integration/tasks/url/impl/validate/helper.py @@ -3,13 +3,13 @@ from src.api.endpoints.annotate.agency.post.dto import URLAgencyAnnotationPostInfo from src.core.enums import RecordType from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource from src.db.models.impl.flag.auto_validated.sqlalchemy import FlagURLAutoValidated from src.db.models.impl.flag.url_validated.enums import URLType from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency from src.db.models.impl.url.core.sqlalchemy import URL from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource from src.db.queries.implementations.anonymous_session import MakeAnonymousSessionQueryBuilder from tests.conftest import db_data_creator from tests.helpers.counter import next_int diff --git a/tests/automated/integration/tasks/url/impl/validate/test_data_source.py b/tests/automated/integration/tasks/url/impl/validate/test_data_source.py index 434e8f06..d99e4448 100644 --- a/tests/automated/integration/tasks/url/impl/validate/test_data_source.py +++ b/tests/automated/integration/tasks/url/impl/validate/test_data_source.py @@ -12,12 +12,12 @@ from src.core.enums import RecordType from src.core.tasks.url.operators.validate.core import AutoValidateURLTaskOperator +from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon +from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon +from src.db.models.impl.annotation.record_type.anon.sqlalchemy import AnnotationAnonRecordType +from src.db.models.impl.annotation.url_type.anon.sqlalchemy import AnnotationAnonURLType from src.db.models.impl.flag.url_validated.enums import URLType -from src.db.models.impl.link.anonymous_sessions__name_suggestion import LinkAnonymousSessionNameSuggestion -from src.db.models.impl.url.suggestion.anonymous.agency.sqlalchemy import AnonymousAnnotationAgency -from src.db.models.impl.url.suggestion.anonymous.location.sqlalchemy import AnonymousAnnotationLocation -from src.db.models.impl.url.suggestion.anonymous.record_type.sqlalchemy import AnonymousAnnotationRecordType -from src.db.models.impl.url.suggestion.anonymous.url_type.sqlalchemy import AnonymousAnnotationURLType +from src.db.models.impl.annotation.name.anon.sqlalchemy import AnnotationNameAnonEndorsement from tests.automated.integration.tasks.url.impl.validate.helper import TestValidateTaskHelper, DEFAULT_RECORD_TYPE from tests.helpers.run import run_task_and_confirm_success @@ -55,27 +55,27 @@ async def test_data_source( session_id_2: UUID = await helper.get_anonymous_session_id() for session_id in [session_id_1, session_id_2]: - anon_url_type = AnonymousAnnotationURLType( + anon_url_type = AnnotationAnonURLType( url_type=URLType.DATA_SOURCE, session_id=session_id, url_id=helper.url_id ) - anon_record_type = AnonymousAnnotationRecordType( + anon_record_type = AnnotationAnonRecordType( record_type=DEFAULT_RECORD_TYPE, session_id=session_id, url_id=helper.url_id ) - anon_location = AnonymousAnnotationLocation( + anon_location = AnnotationLocationAnon( location_id=helper.location_id, session_id=session_id, url_id=helper.url_id ) - anon_agency = AnonymousAnnotationAgency( + anon_agency = AnnotationAgencyAnon( agency_id=helper.agency_id, session_id=session_id, url_id=helper.url_id ) - anon_name_link = LinkAnonymousSessionNameSuggestion( + anon_name_link = AnnotationNameAnonEndorsement( suggestion_id=suggestion_id, session_id=session_id ) @@ -101,7 +101,7 @@ async def test_data_source( # Add tiebreaker -- a single anonymous vote session_id_3: UUID = await helper.get_anonymous_session_id() - anon_record_type = AnonymousAnnotationRecordType( + anon_record_type = AnnotationAnonRecordType( record_type=DEFAULT_RECORD_TYPE, session_id=session_id_3, url_id=helper.url_id diff --git a/tests/helpers/data_creator/commands/impl/suggestion/auto/agency_/core.py b/tests/helpers/data_creator/commands/impl/suggestion/auto/agency_/core.py index fe54c6f9..ab29a817 100644 --- a/tests/helpers/data_creator/commands/impl/suggestion/auto/agency_/core.py +++ b/tests/helpers/data_creator/commands/impl/suggestion/auto/agency_/core.py @@ -3,11 +3,10 @@ from typing_extensions import override from src.core.enums import SuggestionType -from src.core.tasks.url.operators.agency_identification.dtos.suggestion import URLAgencySuggestionInfo from src.db.enums import TaskType -from src.db.models.impl.url.suggestion.agency.subtask.enum import AutoAgencyIDSubtaskType -from src.db.models.impl.url.suggestion.agency.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic -from src.db.models.impl.url.suggestion.agency.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic +from src.db.models.impl.annotation.agency.auto.subtask.enum import AutoAgencyIDSubtaskType +from src.db.models.impl.annotation.agency.auto.subtask.pydantic import URLAutoAgencyIDSubtaskPydantic +from src.db.models.impl.annotation.agency.auto.suggestion.pydantic import AgencyIDSubtaskSuggestionPydantic from tests.helpers.data_creator.commands.base import DBDataCreatorCommandBase from tests.helpers.data_creator.commands.impl.agency import AgencyCommand diff --git a/tests/helpers/data_creator/commands/impl/suggestion/auto/relevant.py b/tests/helpers/data_creator/commands/impl/suggestion/auto/relevant.py index d85b5a1b..498f736c 100644 --- a/tests/helpers/data_creator/commands/impl/suggestion/auto/relevant.py +++ b/tests/helpers/data_creator/commands/impl/suggestion/auto/relevant.py @@ -1,4 +1,4 @@ -from src.db.models.impl.url.suggestion.url_type.auto.pydantic.input import AutoRelevancyAnnotationInput +from src.db.models.impl.annotation.url_type.auto.pydantic.input import AutoRelevancyAnnotationInput from tests.helpers.data_creator.commands.base import DBDataCreatorCommandBase diff --git a/tests/helpers/data_creator/core.py b/tests/helpers/data_creator/core.py index 93ece6e1..d3252202 100644 --- a/tests/helpers/data_creator/core.py +++ b/tests/helpers/data_creator/core.py @@ -14,23 +14,23 @@ from src.db.enums import TaskType from src.db.models.impl.agency.enums import AgencyType, JurisdictionType from src.db.models.impl.agency.sqlalchemy import Agency +from src.db.models.impl.annotation.location.auto.subtask.enums import LocationIDSubtaskType +from src.db.models.impl.annotation.location.auto.subtask.sqlalchemy import AnnotationLocationAutoSubtask +from src.db.models.impl.annotation.location.auto.suggestion.sqlalchemy import AnnotationLocationAutoSuggestion +from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser +from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource +from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion from src.db.models.impl.duplicate.pydantic.insert import DuplicateInsertInfo from src.db.models.impl.flag.root_url.sqlalchemy import FlagRootURL from src.db.models.impl.flag.url_validated.enums import URLType from src.db.models.impl.link.agency_location.sqlalchemy import LinkAgencyLocation from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency from src.db.models.impl.link.urls_root_url.sqlalchemy import LinkURLRootURL -from src.db.models.impl.link.user_name_suggestion.sqlalchemy import LinkUserNameSuggestion +from src.db.models.impl.annotation.name.user.sqlalchemy import LinkUserNameSuggestion from src.db.models.impl.link.user_suggestion_not_found.agency.sqlalchemy import LinkUserSuggestionAgencyNotFound from src.db.models.impl.link.user_suggestion_not_found.location.sqlalchemy import LinkUserSuggestionLocationNotFound from src.db.models.impl.url.core.enums import URLSource from src.db.models.impl.url.html.compressed.sqlalchemy import URLCompressedHTML -from src.db.models.impl.url.suggestion.location.auto.subtask.enums import LocationIDSubtaskType -from src.db.models.impl.url.suggestion.location.auto.subtask.sqlalchemy import AutoLocationIDSubtask -from src.db.models.impl.url.suggestion.location.auto.suggestion.sqlalchemy import LocationIDSubtaskSuggestion -from src.db.models.impl.url.suggestion.location.user.sqlalchemy import UserLocationSuggestion -from src.db.models.impl.url.suggestion.name.enums import NameSuggestionSource -from src.db.models.impl.url.suggestion.name.sqlalchemy import URLNameSuggestion from src.db.models.impl.url.task_error.pydantic_.insert import URLTaskErrorPydantic from src.db.models.impl.url.web_metadata.sqlalchemy import URLWebMetadata from tests.helpers.batch_creation_parameters.core import TestBatchCreationParameters @@ -640,7 +640,7 @@ async def add_user_location_suggestion( user_id: int, location_id: int, ): - suggestion = UserLocationSuggestion( + suggestion = AnnotationLocationUser( url_id=url_id, user_id=user_id, location_id=location_id, @@ -656,7 +656,7 @@ async def add_location_suggestion( ) -> None: locations_found: bool = len(location_ids) > 0 task_id: int = await self.task(url_ids=[url_id]) - subtask = AutoLocationIDSubtask( + subtask = AnnotationLocationAutoSubtask( url_id=url_id, type=type_, task_id=task_id, @@ -665,9 +665,9 @@ async def add_location_suggestion( subtask_id: int = await self.adb_client.add(subtask, return_id=True) if not locations_found: return - suggestions: list[LocationIDSubtaskSuggestion] = [] + suggestions: list[AnnotationLocationAutoSuggestion] = [] for location_id in location_ids: - suggestion = LocationIDSubtaskSuggestion( + suggestion = AnnotationLocationAutoSuggestion( subtask_id=subtask_id, location_id=location_id, confidence=confidence @@ -697,7 +697,7 @@ async def name_suggestion( ) -> int: if name is None: name = f"Test Name {next_int()}" - suggestion = URLNameSuggestion( + suggestion = AnnotationNameSuggestion( url_id=url_id, source=source, suggestion=name,