From 72ed9c9d0ae4a5f42e2f17c7536eff064287ba9b Mon Sep 17 00:00:00 2001 From: maxachis Date: Mon, 21 Apr 2025 16:40:12 -0400 Subject: [PATCH 01/10] DRAFT --- local_database/docker/initdb.d/create-dbs.sql | 3 ++ local_database/docker/initdb.d/setup-fdw.sql | 15 ++++++++ local_database/setup_fdw.sh | 38 +++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 local_database/docker/initdb.d/create-dbs.sql create mode 100644 local_database/docker/initdb.d/setup-fdw.sql create mode 100644 local_database/setup_fdw.sh diff --git a/local_database/docker/initdb.d/create-dbs.sql b/local_database/docker/initdb.d/create-dbs.sql new file mode 100644 index 00000000..1c66dec9 --- /dev/null +++ b/local_database/docker/initdb.d/create-dbs.sql @@ -0,0 +1,3 @@ +-- Creates both logical DBs in one Postgres cluster +CREATE DATABASE data_sources_test_db; +CREATE DATABASE source_collector_test_db; diff --git a/local_database/docker/initdb.d/setup-fdw.sql b/local_database/docker/initdb.d/setup-fdw.sql new file mode 100644 index 00000000..1dd94b4c --- /dev/null +++ b/local_database/docker/initdb.d/setup-fdw.sql @@ -0,0 +1,15 @@ +-- This script connects to db_b and sets up FDW access to db_a +\connect source_collector_test_db; + +CREATE EXTENSION IF NOT EXISTS postgres_fdw; + +CREATE SERVER db_a_server + FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (host 'localhost', dbname 'db_a'); + +CREATE USER MAPPING FOR test_source_collector_user + SERVER db_a_server + OPTIONS (user 'test_source_collector_user', password 'HanviliciousHamiltonHilltops'); + +-- Example: import tables from db_a (assuming public schema exists and has tables) +IMPORT FOREIGN SCHEMA public FROM SERVER db_a_server INTO foreign_a; diff --git a/local_database/setup_fdw.sh b/local_database/setup_fdw.sh new file mode 100644 index 00000000..139dedc7 --- /dev/null +++ b/local_database/setup_fdw.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -euo pipefail + +# Defaults (can be overridden) +POSTGRES_HOST="${POSTGRES_HOST:-localhost}" +POSTGRES_PORT="${POSTGRES_PORT:-5432}" +POSTGRES_USER="${POSTGRES_USER:-postgres}" +POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-postgres}" +DB_A="${DB_A:-db_a}" +DB_B="${DB_B:-db_b}" + +export PGPASSWORD="$POSTGRES_PASSWORD" + +echo "Creating databases $DB_A and $DB_B..." +psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d postgres -p "$POSTGRES_PORT" -c "CREATE DATABASE $DB_A;" || echo "$DB_A already exists" +psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d postgres -p "$POSTGRES_PORT" -c "CREATE DATABASE $DB_B;" || echo "$DB_B already exists" + +echo "Setting up FDW in $DB_B to access $DB_A..." +psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$DB_B" -p "$POSTGRES_PORT" < Date: Tue, 22 Apr 2025 09:15:03 -0400 Subject: [PATCH 02/10] Refactor Docker Logic and add Data Sources Dumper Logic --- .github/workflows/test_app.yml | 7 + ...3f1272f94b9_set_up_foreign_data_wrapper.py | 250 ++++++++++++++ local_database/DTOs.py | 52 +++ local_database/DataDumper/dump.sh | 25 +- local_database/DockerInfos.py | 83 +++++ local_database/classes/DockerClient.py | 81 +++++ local_database/classes/DockerContainer.py | 28 ++ local_database/classes/DockerManager.py | 73 ++++ local_database/classes/TimestampChecker.py | 32 ++ local_database/classes/__init__.py | 0 local_database/constants.py | 5 + local_database/create_database.py | 65 ++++ local_database/docker/initdb.d/create-dbs.sql | 3 - local_database/docker/initdb.d/setup-fdw.sql | 15 - local_database/dump_data_sources_schema.py | 18 + local_database/local_db_util.py | 18 + local_database/setup.py | 52 +++ local_database/setup_fdw.sh | 38 --- start_mirrored_local_app.py | 322 +----------------- 19 files changed, 797 insertions(+), 370 deletions(-) create mode 100644 alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py create mode 100644 local_database/DTOs.py create mode 100644 local_database/DockerInfos.py create mode 100644 local_database/classes/DockerClient.py create mode 100644 local_database/classes/DockerContainer.py create mode 100644 local_database/classes/DockerManager.py create mode 100644 local_database/classes/TimestampChecker.py create mode 100644 local_database/classes/__init__.py create mode 100644 local_database/constants.py create mode 100644 local_database/create_database.py delete mode 100644 local_database/docker/initdb.d/create-dbs.sql delete mode 100644 local_database/docker/initdb.d/setup-fdw.sql create mode 100644 local_database/dump_data_sources_schema.py create mode 100644 local_database/local_db_util.py create mode 100644 local_database/setup.py delete mode 100644 local_database/setup_fdw.sh diff --git a/.github/workflows/test_app.yml b/.github/workflows/test_app.yml index e16d1771..28a41e29 100644 --- a/.github/workflows/test_app.yml +++ b/.github/workflows/test_app.yml @@ -35,6 +35,13 @@ jobs: --health-retries 5 steps: + - name: Set up FDW + run: | + ./local_database/setup_fdw.sh + env: + POSTGRES_HOST: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres - name: Checkout repository uses: actions/checkout@v4 - name: Install dependencies diff --git a/alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py b/alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py new file mode 100644 index 00000000..5c1adf18 --- /dev/null +++ b/alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py @@ -0,0 +1,250 @@ +"""Set up foreign data wrapper + +Revision ID: 13f1272f94b9 +Revises: e285e6e7cf71 +Create Date: 2025-04-21 18:17:34.593973 + +""" +import os +from typing import Sequence, Union + +from alembic import op +from dotenv import load_dotenv + +# revision identifiers, used by Alembic. +revision: str = '13f1272f94b9' +down_revision: Union[str, None] = 'e285e6e7cf71' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + + load_dotenv() + remote_host = os.getenv("DATA_SOURCES_HOST") + user = os.getenv("DATA_SOURCES_USER") + password = os.getenv("DATA_SOURCES_PASSWORD") + db_name = os.getenv("DATA_SOURCES_DB") + port = os.getenv("DATA_SOURCES_PORT") + + op.execute(f"CREATE EXTENSION IF NOT EXISTS postgres_fdw;") + + op.execute(f""" + CREATE SERVER data_sources_server + FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (host '{remote_host}', dbname '{db_name}', port '{port}'); + """) + + op.execute(f""" + CREATE USER MAPPING FOR {user} + SERVER data_sources_server + OPTIONS (user '{user}', password '{password}'); + """) + + op.execute('CREATE SCHEMA if not exists "remote";') + + # Users table + op.execute(""" + CREATE FOREIGN TABLE IF NOT EXISTS "remote".users + ( + id bigint, + created_at timestamp with time zone, + updated_at timestamp with time zone, + email text, + password_digest text, + api_key character varying, + role text + ) + SERVER data_sources_server + OPTIONS ( + schema_name 'public', + table_name 'users' + ); + """) + + # Agencies + # -Enums + # --Jurisdiction Type + op.execute(""" + CREATE TYPE jurisdiction_type AS ENUM + ('school', 'county', 'local', 'port', 'tribal', 'transit', 'state', 'federal'); + """) + # --Agency Type + op.execute(""" + CREATE TYPE agency_type AS ENUM + ('incarceration', 'law enforcement', 'aggregated', 'court', 'unknown'); + """) + + # -Table + op.execute(""" + CREATE FOREIGN TABLE IF NOT EXISTS "remote".agencies + ( + name character , + homepage_url character , + jurisdiction_type jurisdiction_type , + lat double precision, + lng double precision, + defunct_year character , + airtable_uid character , + agency_type agency_type , + multi_agency boolean , + no_web_presence boolean , + airtable_agency_last_modified timestamp with time zone, + rejection_reason character , + last_approval_editor character , + submitter_contact character, + agency_created timestamp with time zone, + id integer, + approval_status text, + creator_user_id integer + ) + SERVER data_sources_server + OPTIONS ( + schema_name 'public', + table_name 'agencies' + ); + """) + + # Locations Table + # -Enums + # --Location Type + op.execute(""" + CREATE TYPE location_type AS ENUM + ('State', 'County', 'Locality'); + """) + + # -Table + op.execute(""" + CREATE FOREIGN TABLE IF NOT EXISTS "remote".locations + ( + id bigint, + type location_type, + state_id bigint, + county_id bigint, + locality_id bigint + ) + SERVER data_sources_server + OPTIONS ( + schema_name 'public', + table_name 'locations' + ); + """) + + # Data Sources Table + + # -Enums + # -- access_type + op.execute(""" + CREATE TYPE access_type AS ENUM + ('Download', 'Webpage', 'API'); + """) + + # -- agency_aggregation + op.execute(""" + CREATE TYPE agency_aggregation AS ENUM + ('county', 'local', 'state', 'federal'); + """) + # -- update_method + op.execute(""" + CREATE TYPE update_method AS ENUM + ('Insert', 'No updates', 'Overwrite'); + """) + + # -- detail_level + op.execute(""" + CREATE TYPE detail_level AS ENUM + ('Individual record', 'Aggregated records', 'Summarized totals'); + """) + + # -- retention_schedule + op.execute(""" + CREATE TYPE retention_schedule AS ENUM + ('< 1 day', '1 day', '< 1 week', '1 week', '1 month', '< 1 year', '1-10 years', '> 10 years', 'Future only'); + """) + + # -Table + op.execute(""" + CREATE FOREIGN TABLE IF NOT EXISTS "remote".data_sources + ( + name character varying , + description character , + source_url character , + agency_supplied boolean, + supplying_entity character , + agency_originated boolean, + agency_aggregation agency_aggregation, + coverage_start date, + coverage_end date, + updated_at timestamp with time zone , + detail_level detail_level, + record_download_option_provided boolean, + data_portal_type character , + update_method update_method, + readme_url character , + originating_entity character , + retention_schedule retention_schedule, + airtable_uid character , + scraper_url character , + created_at timestamp with time zone , + submission_notes character , + rejection_note character , + submitter_contact_info character , + agency_described_not_in_database character , + data_portal_type_other character , + data_source_request character , + broken_source_url_as_of timestamp with time zone, + access_notes text , + url_status text , + approval_status text , + record_type_id integer, + access_types access_type[], + tags text[] , + record_formats text[] , + id integer, + approval_status_updated_at timestamp with time zone , + last_approval_editor bigint + ) + SERVER data_sources_server + OPTIONS ( + schema_name 'public', + table_name 'data_sources' + ); + """) + + + +def downgrade() -> None: + # Drop foreign schema + op.execute('DROP SCHEMA IF EXISTS "remote" CASCADE;') + + # Drop enums + enums = [ + "jurisdiction_type", + "agency_type", + "location_type", + "access_type", + "agency_aggregation", + "update_method", + "detail_level", + "retention_schedule", + ] + for enum in enums: + op.execute(f""" + DROP TYPE IF EXISTS {enum}; + """) + + # Drop user mapping + user = os.getenv("DATA_SOURCES_USER") + op.execute(f""" + DROP USER MAPPING FOR {user} SERVER data_sources_server; + """) + + # Drop server + op.execute(""" + DROP SERVER IF EXISTS data_sources_server CASCADE; + """) + + # Drop FDW + op.execute(""" + DROP EXTENSION IF EXISTS postgres_fdw CASCADE; + """) diff --git a/local_database/DTOs.py b/local_database/DTOs.py new file mode 100644 index 00000000..c4c5ff80 --- /dev/null +++ b/local_database/DTOs.py @@ -0,0 +1,52 @@ +from typing import Annotated, Optional + +from pydantic import BaseModel, AfterValidator + +from local_database.local_db_util import is_absolute_path, get_absolute_path + + +class VolumeInfo(BaseModel): + host_path: str + container_path: Annotated[str, AfterValidator(is_absolute_path)] + + def build_volumes(self): + return { + get_absolute_path(self.host_path): { + "bind": self.container_path, + "mode": "rw" + } + } + + +class DockerfileInfo(BaseModel): + image_tag: str + dockerfile_directory: Optional[str] = None + + +class HealthCheckInfo(BaseModel): + test: list[str] + interval: int + timeout: int + retries: int + start_period: int + + def build_healthcheck(self) -> dict: + multiplicative_factor = 1000000000 # Assume 1 second + return { + "test": self.test, + "interval": self.interval * multiplicative_factor, + "timeout": self.timeout * multiplicative_factor, + "retries": self.retries, + "start_period": self.start_period * multiplicative_factor + } + + +class DockerInfo(BaseModel): + dockerfile_info: DockerfileInfo + volume_info: Optional[VolumeInfo] = None + name: str + ports: Optional[dict] = None + environment: Optional[dict] + command: Optional[str] = None + entrypoint: Optional[list[str]] = None + health_check_info: Optional[HealthCheckInfo] = None diff --git a/local_database/DataDumper/dump.sh b/local_database/DataDumper/dump.sh index 9c07c0ca..482a3ca1 100644 --- a/local_database/DataDumper/dump.sh +++ b/local_database/DataDumper/dump.sh @@ -1,15 +1,28 @@ #!/bin/bash #set -e + # Variables (customize these or pass them as environment variables) DB_HOST=${DUMP_HOST:-"postgres_container"} DB_USER=${DUMP_USER:-"your_user"} -DB_PORT=${DUMP_PORT:-"5432"} # Default to 5432 if not provided +DB_PORT=${DUMP_PORT:-"5432"} DB_PASSWORD=${DUMP_PASSWORD:-"your_password"} DB_NAME=${DUMP_NAME:-"your_database"} -DUMP_FILE="/dump/db_dump.sql" +DUMP_FILE=${DUMP_FILE:-"/dump/db_dump.sql"} +DUMP_SCHEMA_ONLY=${DUMP_SCHEMA_ONLY:-false} # Set to "true" to dump only schema + # Export password for pg_dump export PGPASSWORD=$DB_PASSWORD -# Dump the database -echo "Dumping database $DB_NAME from $DB_HOST:$DB_PORT..." -pg_dump -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME --no-owner --no-acl -F c -f $DUMP_FILE -echo "Dump completed. File saved to $DUMP_FILE." \ No newline at end of file + +# Determine pg_dump flags +PG_DUMP_FLAGS="--no-owner --no-acl -F c" +if [[ "$DUMP_SCHEMA_ONLY" == "true" ]]; then + PG_DUMP_FLAGS="$PG_DUMP_FLAGS --schema-only" + echo "Dumping schema only..." +else + echo "Dumping full database..." +fi + +# Run pg_dump +pg_dump -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME $PG_DUMP_FLAGS -f $DUMP_FILE + +echo "Dump completed. File saved to $DUMP_FILE." diff --git a/local_database/DockerInfos.py b/local_database/DockerInfos.py new file mode 100644 index 00000000..aecff2b7 --- /dev/null +++ b/local_database/DockerInfos.py @@ -0,0 +1,83 @@ +from local_database.DTOs import DockerInfo, DockerfileInfo, HealthCheckInfo, VolumeInfo +from local_database.constants import LOCAL_DATA_SOURCES_DB_NAME +from util.helper_functions import get_from_env + + +def get_database_docker_info() -> DockerInfo: + return DockerInfo( + dockerfile_info=DockerfileInfo( + image_tag="postgres:15", + ), + name="data_source_identification_db", + ports={ + "5432/tcp": 5432 + }, + environment={ + "POSTGRES_PASSWORD": "HanviliciousHamiltonHilltops", + "POSTGRES_USER": "test_source_collector_user", + "POSTGRES_DB": "source_collector_test_db" + }, + health_check_info=HealthCheckInfo( + test=["pg_isready", "-U", "test_source_collector_user", "-h", "127.0.0.1", "-p", "5432"], + interval=1, + timeout=3, + retries=30, + start_period=2 + ) + ) + + +def get_data_sources_data_dumper_info() -> DockerInfo: + return DockerInfo( + dockerfile_info=DockerfileInfo( + image_tag="datadumper", + dockerfile_directory="DataDumper" + ), + volume_info=VolumeInfo( + host_path="./DataDumper/dump", + container_path="/dump" + ), + name="datadumper", + environment={ + "DUMP_HOST": get_from_env("PROD_DATA_SOURCES_HOST"), + "DUMP_USER": get_from_env("PROD_DATA_SOURCES_USER"), + "DUMP_PASSWORD": get_from_env("PROD_DATA_SOURCES_PASSWORD"), + "DUMP_NAME": get_from_env("PROD_DATA_SOURCES_DB"), + "DUMP_PORT": get_from_env("PROD_DATA_SOURCES_PORT"), + "RESTORE_HOST": get_from_env("POSTGRES_HOST"), + "RESTORE_USER": get_from_env("POSTGRES_USER"), + "RESTORE_PORT": get_from_env("POSTGRES_PORT"), + "RESTORE_DB_NAME": LOCAL_DATA_SOURCES_DB_NAME, + "RESTORE_PASSWORD": get_from_env("POSTGRES_PASSWORD"), + "DUMP_FILE": "/dump/data_sources_db_dump.sql", + "DUMP_SCHEMA_ONLY": "true" + }, + command="bash" + ) + + +def get_source_collector_data_dumper_info() -> DockerInfo: + return DockerInfo( + dockerfile_info=DockerfileInfo( + image_tag="datadumper", + dockerfile_directory="DataDumper" + ), + volume_info=VolumeInfo( + host_path="./DataDumper/dump", + container_path="/dump" + ), + name="datadumper", + environment={ + "DUMP_HOST": get_from_env("DUMP_HOST"), + "DUMP_USER": get_from_env("DUMP_USER"), + "DUMP_PASSWORD": get_from_env("DUMP_PASSWORD"), + "DUMP_NAME": get_from_env("DUMP_DB_NAME"), + "DUMP_PORT": get_from_env("DUMP_PORT"), + "RESTORE_HOST": "data_source_identification_db", + "RESTORE_USER": "test_source_collector_user", + "RESTORE_PORT": "5432", + "RESTORE_DB_NAME": "source_collector_test_db", + "RESTORE_PASSWORD": "HanviliciousHamiltonHilltops", + }, + command="bash" + ) diff --git a/local_database/classes/DockerClient.py b/local_database/classes/DockerClient.py new file mode 100644 index 00000000..bb452748 --- /dev/null +++ b/local_database/classes/DockerClient.py @@ -0,0 +1,81 @@ +import docker +from docker.errors import NotFound, APIError + +from local_database.DTOs import DockerfileInfo, DockerInfo +from local_database.local_db_util import get_absolute_path + + +class DockerClient: + + def __init__(self): + self.client = docker.from_env() + + def run_command(self, command: str, container_id: str): + exec_id = self.client.api.exec_create( + container_id, + cmd=command, + tty=True, + stdin=False + ) + output_stream = self.client.api.exec_start(exec_id=exec_id, stream=True) + for line in output_stream: + print(line.decode().rstrip()) + + def start_network(self, network_name): + try: + self.client.networks.create(network_name, driver="bridge") + except APIError as e: + # Assume already exists + print(e) + return self.client.networks.get(network_name) + + def stop_network(self, network_name): + self.client.networks.get(network_name).remove() + + def get_image(self, dockerfile_info: DockerfileInfo): + if dockerfile_info.dockerfile_directory: + # Build image from Dockerfile + self.client.images.build( + path=get_absolute_path(dockerfile_info.dockerfile_directory), + tag=dockerfile_info.image_tag + ) + else: + # Pull or use existing image + self.client.images.pull(dockerfile_info.image_tag) + + def run_container( + self, + docker_info: DockerInfo, + network_name: str + ): + print(f"Running container {docker_info.name}") + try: + container = self.client.containers.get(docker_info.name) + if container.status == 'running': + print(f"Container '{docker_info.name}' is already running") + return container + print("Restarting container...") + container.start() + return container + except NotFound: + # Container does not exist; proceed to build/pull image and run + pass + + self.get_image(docker_info.dockerfile_info) + + container = self.client.containers.run( + image=docker_info.dockerfile_info.image_tag, + volumes=docker_info.volume_info.build_volumes() if docker_info.volume_info is not None else None, + command=docker_info.command, + entrypoint=docker_info.entrypoint, + detach=True, + name=docker_info.name, + ports=docker_info.ports, + network=network_name, + environment=docker_info.environment, + stdout=True, + stderr=True, + tty=True, + healthcheck=docker_info.health_check_info.build_healthcheck() if docker_info.health_check_info is not None else None + ) + return container diff --git a/local_database/classes/DockerContainer.py b/local_database/classes/DockerContainer.py new file mode 100644 index 00000000..ee2ecba9 --- /dev/null +++ b/local_database/classes/DockerContainer.py @@ -0,0 +1,28 @@ +import time + +from docker.models.containers import Container + +from local_database.classes.DockerClient import DockerClient + + +class DockerContainer: + + def __init__(self, dc: DockerClient, container: Container): + self.dc = dc + self.container = container + + def run_command(self, command: str): + self.dc.run_command(command, self.container.id) + + def stop(self): + self.container.stop() + + def wait_for_pg_to_be_ready(self): + for i in range(30): + exit_code, output = self.container.exec_run("pg_isready") + print(output) + if exit_code == 0: + return + time.sleep(1) + raise Exception("Timed out waiting for postgres to be ready") + diff --git a/local_database/classes/DockerManager.py b/local_database/classes/DockerManager.py new file mode 100644 index 00000000..ab43f852 --- /dev/null +++ b/local_database/classes/DockerManager.py @@ -0,0 +1,73 @@ +import platform +import subprocess +import sys + +import docker +from docker.errors import APIError + +from local_database.DTOs import DockerfileInfo, DockerInfo +from local_database.classes.DockerClient import DockerClient +from local_database.classes.DockerContainer import DockerContainer + + +class DockerManager: + def __init__(self): + if not self.is_docker_running(): + self.start_docker_engine() + + self.client = DockerClient() + self.network_name = "my_network" + self.network = self.start_network() + + @staticmethod + def start_docker_engine(): + system = platform.system() + + match system: + case "Windows": + # Use PowerShell to start Docker Desktop on Windows + subprocess.run([ + "powershell", "-Command", + "Start-Process 'Docker Desktop' -Verb RunAs" + ]) + case "Darwin": + # MacOS: Docker Desktop must be started manually or with open + subprocess.run(["open", "-a", "Docker"]) + case "Linux": + # Most Linux systems use systemctl to manage Docker + subprocess.run(["sudo", "systemctl", "start", "docker"]) + case _: + print(f"Unsupported OS: {system}") + sys.exit(1) + + @staticmethod + def is_docker_running(): + try: + client = docker.from_env() + client.ping() + return True + except docker.errors.DockerException as e: + print(f"Docker is not running: {e}") + return False + + def run_command(self, command: str, container_id: str): + self.client.run_command(command, container_id) + + def start_network(self): + return self.client.start_network(self.network_name) + + def stop_network(self): + self.client.stop_network(self.network_name) + + def get_image(self, dockerfile_info: DockerfileInfo): + self.client.get_image(dockerfile_info) + + def run_container( + self, + docker_info: DockerInfo, + ) -> DockerContainer: + raw_container = self.client.run_container(docker_info, self.network_name) + return DockerContainer(self.client, raw_container) + + def get_containers(self): + return self.client.client.containers.list() \ No newline at end of file diff --git a/local_database/classes/TimestampChecker.py b/local_database/classes/TimestampChecker.py new file mode 100644 index 00000000..56779fd4 --- /dev/null +++ b/local_database/classes/TimestampChecker.py @@ -0,0 +1,32 @@ +import datetime +import os +from typing import Optional + + +class TimestampChecker: + def __init__(self): + self.last_run_time: Optional[datetime.datetime] = self.load_last_run_time() + + def load_last_run_time(self) -> Optional[datetime.datetime]: + # Check if file `last_run.txt` exists + # If it does, load the last run time + if os.path.exists("local_state/last_run.txt"): + with open("local_state/last_run.txt", "r") as f: + return datetime.datetime.strptime( + f.read(), + "%Y-%m-%d %H:%M:%S" + ) + return None + + def last_run_within_24_hours(self): + if self.last_run_time is None: + return False + return datetime.datetime.now() - self.last_run_time < datetime.timedelta(days=1) + + def set_last_run_time(self): + # If directory `local_state` doesn't exist, create it + if not os.path.exists("local_state"): + os.makedirs("local_state") + + with open("local_state/last_run.txt", "w") as f: + f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) diff --git a/local_database/classes/__init__.py b/local_database/classes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_database/constants.py b/local_database/constants.py new file mode 100644 index 00000000..d5c96e72 --- /dev/null +++ b/local_database/constants.py @@ -0,0 +1,5 @@ +LOCAL_DATA_SOURCES_DB_NAME = "test_data_sources_db" +LOCAL_SOURCE_COLLECTOR_DB_NAME = "source_collector_test_db" + +DUMP_SH_DOCKER_PATH = "/usr/local/bin/dump.sh" +RESTORE_SH_DOCKER_PATH = "/usr/local/bin/restore.sh" \ No newline at end of file diff --git a/local_database/create_database.py b/local_database/create_database.py new file mode 100644 index 00000000..b23cc6d2 --- /dev/null +++ b/local_database/create_database.py @@ -0,0 +1,65 @@ +import os +import psycopg2 +from psycopg2 import sql + +from local_database.constants import LOCAL_DATA_SOURCES_DB_NAME, LOCAL_SOURCE_COLLECTOR_DB_NAME + +# Defaults (can be overridden via environment variables) +POSTGRES_HOST = os.getenv("POSTGRES_HOST", "host.docker.internal") +POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432")) +POSTGRES_USER = os.getenv("POSTGRES_USER", "test_source_collector_user") +POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", "HanviliciousHamiltonHilltops") + + +# Connect to the default 'postgres' database to create other databases +def connect(database="postgres", autocommit=True): + conn = psycopg2.connect( + dbname=database, + user=POSTGRES_USER, + password=POSTGRES_PASSWORD, + host=POSTGRES_HOST, + port=POSTGRES_PORT + ) + if autocommit: + conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + return conn + +def create_database(db_name): + conn = connect("postgres") + with conn.cursor() as cur: + cur.execute(sql.SQL(""" + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = %s AND pid <> pg_backend_pid() + """), [db_name]) + + # Drop the database if it exists + cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(db_name))) + print(f"🗑️ Dropped existing database: {db_name}") + + try: + cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(db_name))) + print(f"✅ Created database: {db_name}") + except psycopg2.errors.DuplicateDatabase: + print(f"⚠️ Database {db_name} already exists") + except Exception as e: + print(f"❌ Failed to create {db_name}: {e}") + +def create_database_tables(): + conn = connect(LOCAL_DATA_SOURCES_DB_NAME) + with conn.cursor() as cur: + cur.execute(""" + CREATE TABLE IF NOT EXISTS test_table ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL + ) + """) + conn.commit() + +def main(): + print("Creating databases...") + create_database(LOCAL_DATA_SOURCES_DB_NAME) + create_database(LOCAL_SOURCE_COLLECTOR_DB_NAME) + +if __name__ == "__main__": + main() diff --git a/local_database/docker/initdb.d/create-dbs.sql b/local_database/docker/initdb.d/create-dbs.sql deleted file mode 100644 index 1c66dec9..00000000 --- a/local_database/docker/initdb.d/create-dbs.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Creates both logical DBs in one Postgres cluster -CREATE DATABASE data_sources_test_db; -CREATE DATABASE source_collector_test_db; diff --git a/local_database/docker/initdb.d/setup-fdw.sql b/local_database/docker/initdb.d/setup-fdw.sql deleted file mode 100644 index 1dd94b4c..00000000 --- a/local_database/docker/initdb.d/setup-fdw.sql +++ /dev/null @@ -1,15 +0,0 @@ --- This script connects to db_b and sets up FDW access to db_a -\connect source_collector_test_db; - -CREATE EXTENSION IF NOT EXISTS postgres_fdw; - -CREATE SERVER db_a_server - FOREIGN DATA WRAPPER postgres_fdw - OPTIONS (host 'localhost', dbname 'db_a'); - -CREATE USER MAPPING FOR test_source_collector_user - SERVER db_a_server - OPTIONS (user 'test_source_collector_user', password 'HanviliciousHamiltonHilltops'); - --- Example: import tables from db_a (assuming public schema exists and has tables) -IMPORT FOREIGN SCHEMA public FROM SERVER db_a_server INTO foreign_a; diff --git a/local_database/dump_data_sources_schema.py b/local_database/dump_data_sources_schema.py new file mode 100644 index 00000000..49627bc3 --- /dev/null +++ b/local_database/dump_data_sources_schema.py @@ -0,0 +1,18 @@ +from local_database.DockerInfos import get_data_sources_data_dumper_info +from local_database.classes.DockerManager import DockerManager +from local_database.constants import DUMP_SH_DOCKER_PATH + + +def main(): + docker_manager = DockerManager() + data_sources_docker_info = get_data_sources_data_dumper_info() + container = docker_manager.run_container(data_sources_docker_info) + try: + container.run_command(DUMP_SH_DOCKER_PATH) + finally: + container.stop() + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/local_database/local_db_util.py b/local_database/local_db_util.py new file mode 100644 index 00000000..7bc5bb12 --- /dev/null +++ b/local_database/local_db_util.py @@ -0,0 +1,18 @@ +from pathlib import Path + + +def get_absolute_path(relative_path: str) -> str: + """ + Get absolute path, using the current file as the point of reference + """ + current_dir = Path(__file__).parent + absolute_path = (current_dir / relative_path).resolve() + return str(absolute_path) + + +def is_absolute_path(path: str) -> str: + if len(path) == 0: + raise ValueError("Path is required") + if path[0] != "/": + raise ValueError("Container path must be absolute") + return path diff --git a/local_database/setup.py b/local_database/setup.py new file mode 100644 index 00000000..a720ebc2 --- /dev/null +++ b/local_database/setup.py @@ -0,0 +1,52 @@ +import subprocess +import time +import sys + +POSTGRES_SERVICE_NAME = "postgres" +FOLLOWUP_SCRIPT = "py create_database.py" +MAX_RETRIES = 20 +SLEEP_SECONDS = 1 + +def run_command(cmd, check=True, capture_output=False, **kwargs): + try: + return subprocess.run(cmd, shell=True, check=check, capture_output=capture_output, text=True, **kwargs) + except subprocess.CalledProcessError as e: + print(f"Command '{cmd}' failed: {e}") + sys.exit(1) + +def get_postgres_container_id(): + result = run_command(f"docker-compose ps -q {POSTGRES_SERVICE_NAME}", capture_output=True) + container_id = result.stdout.strip() + if not container_id: + print("Error: Could not find Postgres container.") + sys.exit(1) + return container_id + +def wait_for_postgres(container_id): + print("Waiting for Postgres to be ready...") + for i in range(MAX_RETRIES): + try: + run_command(f"docker exec {container_id} pg_isready -U postgres", check=True) + print("Postgres is ready!") + return + except subprocess.CalledProcessError: + print(f"Still waiting... ({i+1}/{MAX_RETRIES})") + time.sleep(SLEEP_SECONDS) + print("Postgres did not become ready in time.") + sys.exit(1) + +def main(): + print("Stopping Docker Compose...") + run_command("docker-compose down") + + print("Starting Docker Compose...") + run_command("docker-compose up -d") + + container_id = get_postgres_container_id() + wait_for_postgres(container_id) + + print("Running follow-up script...") + run_command(FOLLOWUP_SCRIPT) + +if __name__ == "__main__": + main() diff --git a/local_database/setup_fdw.sh b/local_database/setup_fdw.sh deleted file mode 100644 index 139dedc7..00000000 --- a/local_database/setup_fdw.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Defaults (can be overridden) -POSTGRES_HOST="${POSTGRES_HOST:-localhost}" -POSTGRES_PORT="${POSTGRES_PORT:-5432}" -POSTGRES_USER="${POSTGRES_USER:-postgres}" -POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-postgres}" -DB_A="${DB_A:-db_a}" -DB_B="${DB_B:-db_b}" - -export PGPASSWORD="$POSTGRES_PASSWORD" - -echo "Creating databases $DB_A and $DB_B..." -psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d postgres -p "$POSTGRES_PORT" -c "CREATE DATABASE $DB_A;" || echo "$DB_A already exists" -psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d postgres -p "$POSTGRES_PORT" -c "CREATE DATABASE $DB_B;" || echo "$DB_B already exists" - -echo "Setting up FDW in $DB_B to access $DB_A..." -psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$DB_B" -p "$POSTGRES_PORT" < str: - if len(path) == 0: - raise ValueError("Path is required") - if path[0] != "/": - raise ValueError("Container path must be absolute") - return path - -class VolumeInfo(BaseModel): - host_path: str - container_path: Annotated[str, AfterValidator(is_absolute_path)] - - def build_volumes(self): - return { - get_absolute_path(self.host_path): { - "bind": self.container_path, - "mode": "rw" - } - } - -def wait_for_pg_to_be_ready(container: Container): - for i in range(30): - exit_code, output = container.exec_run("pg_isready") - print(output) - if exit_code == 0: - return - time.sleep(1) - raise Exception("Timed out waiting for postgres to be ready") - -def get_absolute_path(relative_path: str) -> str: - """ - Get absolute path, using the current file as the point of reference - """ - current_dir = Path(__file__).parent - absolute_path = (current_dir / relative_path).resolve() - return str(absolute_path) - - -class DockerfileInfo(BaseModel): - image_tag: str - dockerfile_directory: Optional[str] = None - - - -class HealthCheckInfo(BaseModel): - test: list[str] - interval: int - timeout: int - retries: int - start_period: int - - def build_healthcheck(self) -> dict: - multiplicative_factor = 1000000000 # Assume 1 second - return { - "test": self.test, - "interval": self.interval * multiplicative_factor, - "timeout": self.timeout * multiplicative_factor, - "retries": self.retries, - "start_period": self.start_period * multiplicative_factor - } - -class DockerInfo(BaseModel): - dockerfile_info: DockerfileInfo - volume_info: Optional[VolumeInfo] = None - name: str - ports: Optional[dict] = None - environment: Optional[dict] - command: Optional[str] = None - entrypoint: Optional[list[str]] = None - health_check_info: Optional[HealthCheckInfo] = None - -def run_command_checked(command: list[str] or str, shell=False): - result = subprocess.run( - command, - check=True, - capture_output=True, - text=True, - shell=shell - ) - return result - -def is_docker_running(): - try: - client = docker.from_env() - client.ping() - return True - except docker.errors.DockerException as e: - print(f"Docker is not running: {e}") - return False - -def wait_for_health(container, timeout=30): - start = time.time() - while time.time() - start < timeout: - container.reload() # Refresh container state - state = container.attrs.get("State") - print(state) - health = container.attrs.get("State", {}).get("Health", {}) - status = health.get("Status") - print(f"Health status: {status}") - if status == "healthy": - print("Postgres is healthy.") - return - elif status == "unhealthy": - raise Exception("Postgres container became unhealthy.") - time.sleep(1) - raise TimeoutError("Timed out waiting for Postgres to become healthy.") - -def start_docker_engine(): - system = platform.system() - - match system: - case "Windows": - # Use PowerShell to start Docker Desktop on Windows - subprocess.run([ - "powershell", "-Command", - "Start-Process 'Docker Desktop' -Verb RunAs" - ]) - case "Darwin": - # MacOS: Docker Desktop must be started manually or with open - subprocess.run(["open", "-a", "Docker"]) - case "Linux": - # Most Linux systems use systemctl to manage Docker - subprocess.run(["sudo", "systemctl", "start", "docker"]) - case _: - print(f"Unsupported OS: {system}") - sys.exit(1) - -class DockerManager: - def __init__(self): - self.client = docker.from_env() - self.network_name = "my_network" - self.network = self.start_network() +from local_database.DockerInfos import get_database_docker_info, get_source_collector_data_dumper_info +from local_database.classes.DockerManager import DockerManager +from local_database.classes.TimestampChecker import TimestampChecker +from local_database.constants import RESTORE_SH_DOCKER_PATH, DUMP_SH_DOCKER_PATH - def run_command(self, command: str, container_id: str): - exec_id = self.client.api.exec_create( - container_id, - cmd=command, - tty=True, - stdin=False - ) - output_stream = self.client.api.exec_start(exec_id=exec_id, stream=True) - for line in output_stream: - print(line.decode().rstrip()) - - def start_network(self): - try: - self.client.networks.create(self.network_name, driver="bridge") - except APIError as e: - # Assume already exists - print(e) - return self.client.networks.get("my_network") - - def stop_network(self): - self.client.networks.get("my_network").remove() - - def get_image(self, dockerfile_info: DockerfileInfo): - if dockerfile_info.dockerfile_directory: - # Build image from Dockerfile - self.client.images.build( - path=get_absolute_path(dockerfile_info.dockerfile_directory), - tag=dockerfile_info.image_tag - ) - else: - # Pull or use existing image - self.client.images.pull(dockerfile_info.image_tag) - - - def run_container( - self, - docker_info: DockerInfo, - ) -> Container: - print(f"Running container {docker_info.name}") - try: - container = self.client.containers.get(docker_info.name) - if container.status == 'running': - print(f"Container '{docker_info.name}' is already running") - return container - print("Restarting container...") - container.start() - return container - except NotFound: - # Container does not exist; proceed to build/pull image and run - pass - - self.get_image(docker_info.dockerfile_info) - - container = self.client.containers.run( - image=docker_info.dockerfile_info.image_tag, - volumes=docker_info.volume_info.build_volumes() if docker_info.volume_info is not None else None, - command=docker_info.command, - entrypoint=docker_info.entrypoint, - detach=True, - name=docker_info.name, - ports=docker_info.ports, - network=self.network_name, - environment=docker_info.environment, - stdout=True, - stderr=True, - tty=True, - healthcheck=docker_info.health_check_info.build_healthcheck() if docker_info.health_check_info is not None else None - ) - return container - - -class TimestampChecker: - def __init__(self): - self.last_run_time: Optional[datetime.datetime] = self.load_last_run_time() - - def load_last_run_time(self) -> Optional[datetime.datetime]: - # Check if file `last_run.txt` exists - # If it does, load the last run time - if os.path.exists("local_state/last_run.txt"): - with open("local_state/last_run.txt", "r") as f: - return datetime.datetime.strptime( - f.read(), - "%Y-%m-%d %H:%M:%S" - ) - return None - - def last_run_within_24_hours(self): - if self.last_run_time is None: - return False - return datetime.datetime.now() - self.last_run_time < datetime.timedelta(days=1) - - def set_last_run_time(self): - # If directory `local_state` doesn't exist, create it - if not os.path.exists("local_state"): - os.makedirs("local_state") - - with open("local_state/last_run.txt", "w") as f: - f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - -def get_database_docker_info() -> DockerInfo: - return DockerInfo( - dockerfile_info=DockerfileInfo( - image_tag="postgres:15", - ), - name="data_source_identification_db", - ports={ - "5432/tcp": 5432 - }, - environment={ - "POSTGRES_PASSWORD": "HanviliciousHamiltonHilltops", - "POSTGRES_USER": "test_source_collector_user", - "POSTGRES_DB": "source_collector_test_db" - }, - health_check_info=HealthCheckInfo( - test=["pg_isready", "-U", "test_source_collector_user", "-h", "127.0.0.1", "-p", "5432"], - interval=1, - timeout=3, - retries=30, - start_period=2 - ) - ) - -def get_data_dumper_docker_info() -> DockerInfo: - return DockerInfo( - dockerfile_info=DockerfileInfo( - image_tag="datadumper", - dockerfile_directory="local_database/DataDumper" - ), - volume_info=VolumeInfo( - host_path="./local_database/DataDumper/dump", - container_path="/dump" - ), - name="datadumper", - environment={ - "DUMP_HOST": get_from_env("DUMP_HOST"), - "DUMP_USER": get_from_env("DUMP_USER"), - "DUMP_PASSWORD": get_from_env("DUMP_PASSWORD"), - "DUMP_NAME": get_from_env("DUMP_DB_NAME"), - "DUMP_PORT": get_from_env("DUMP_PORT"), - "RESTORE_HOST": "data_source_identification_db", - "RESTORE_USER": "test_source_collector_user", - "RESTORE_PORT": "5432", - "RESTORE_DB_NAME": "source_collector_test_db", - "RESTORE_PASSWORD": "HanviliciousHamiltonHilltops", - }, - command="bash" - ) def main(): docker_manager = DockerManager() - # Ensure docker is running, and start if not - if not is_docker_running(): - start_docker_engine() # Ensure Dockerfile for database is running, and if not, start it database_docker_info = get_database_docker_info() - container = docker_manager.run_container(database_docker_info) - wait_for_pg_to_be_ready(container) + db_container = docker_manager.run_container(database_docker_info) + db_container.wait_for_pg_to_be_ready() # Start dockerfile for Datadumper - data_dumper_docker_info = get_data_dumper_docker_info() + data_dumper_docker_info = get_source_collector_data_dumper_info() # If not last run within 24 hours, run dump operation in Datadumper # Check cache if exists and checker = TimestampChecker() - container = docker_manager.run_container(data_dumper_docker_info) + data_dump_container = docker_manager.run_container(data_dumper_docker_info) if checker.last_run_within_24_hours(): print("Last run within 24 hours, skipping dump...") else: - docker_manager.run_command( - '/usr/local/bin/dump.sh', - container.id + data_dump_container.run_command( + DUMP_SH_DOCKER_PATH, ) - docker_manager.run_command( - "/usr/local/bin/restore.sh", - container.id + data_dump_container.run_command( + RESTORE_SH_DOCKER_PATH, ) print("Stopping datadumper container") - container.stop() + data_dump_container.stop() checker.set_last_run_time() # Upgrade using alembic @@ -351,7 +57,7 @@ def main(): finally: # Add feature to stop all running containers print("Stopping containers...") - for container in docker_manager.client.containers.list(): + for container in docker_manager.get_containers(): container.stop() print("Containers stopped.") From ca27fdbe6a2252b0615267687fb10508f9c9e3cf Mon Sep 17 00:00:00 2001 From: maxachis Date: Tue, 22 Apr 2025 11:01:17 -0400 Subject: [PATCH 03/10] add .gitattributes --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..dfdb8b77 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf From 0e1eb394ba580c247c6fb422e92b90847503ce33 Mon Sep 17 00:00:00 2001 From: maxachis Date: Tue, 22 Apr 2025 13:28:05 -0400 Subject: [PATCH 04/10] DRAFT --- ENV.md | 12 +++- local_database/DTOs.py | 2 +- local_database/DockerInfos.py | 24 +++++-- local_database/classes/DockerClient.py | 83 ++++++++++++++++------ local_database/classes/DockerManager.py | 7 +- local_database/dump_data_sources_schema.py | 5 +- util/helper_functions.py | 10 +++ 7 files changed, 111 insertions(+), 32 deletions(-) diff --git a/ENV.md b/ENV.md index 5292320b..3452ef7c 100644 --- a/ENV.md +++ b/ENV.md @@ -21,4 +21,14 @@ Please ensure these are properly defined in a `.env` file in the root directory. |`PDAP_API_URL`| The URL for the PDAP API| `https://data-sources-v2.pdap.dev/api`| |`DISCORD_WEBHOOK_URL`| The URL for the Discord webhook used for notifications| `abc123` | -[^1:] The user account in question will require elevated permissions to access certain endpoints. At a minimum, the user will require the `source_collector` and `db_write` permissions. \ No newline at end of file +[^1:] The user account in question will require elevated permissions to access certain endpoints. At a minimum, the user will require the `source_collector` and `db_write` permissions. + +## Data Dumper + +``` +PROD_DATA_SOURCES_HOST=pdap-production-v2-do-user-8463429-0.k.db.ondigitalocean.com # The host of the production Data Sources Database +PROD_DATA_SOURCES_PORT=25060 # The port of the production Data Sources Database +PROD_DATA_SOURCES_USER=dump_user # The username for the production Data Sources Database +PROD_DATA_SOURCES_PASSWORD=GeriatricGeronimoGentrification # The password for the production Data Sources Database +PROD_DATA_SOURCES_DB=pdap_prod_v2_db # The database name for the production Data Sources Database +``` \ No newline at end of file diff --git a/local_database/DTOs.py b/local_database/DTOs.py index c4c5ff80..f222e5ba 100644 --- a/local_database/DTOs.py +++ b/local_database/DTOs.py @@ -11,7 +11,7 @@ class VolumeInfo(BaseModel): def build_volumes(self): return { - get_absolute_path(self.host_path): { + self.host_path: { "bind": self.container_path, "mode": "rw" } diff --git a/local_database/DockerInfos.py b/local_database/DockerInfos.py index aecff2b7..3b1c071b 100644 --- a/local_database/DockerInfos.py +++ b/local_database/DockerInfos.py @@ -1,6 +1,6 @@ from local_database.DTOs import DockerInfo, DockerfileInfo, HealthCheckInfo, VolumeInfo from local_database.constants import LOCAL_DATA_SOURCES_DB_NAME -from util.helper_functions import get_from_env +from util.helper_functions import get_from_env, project_path def get_database_docker_info() -> DockerInfo: @@ -31,10 +31,17 @@ def get_data_sources_data_dumper_info() -> DockerInfo: return DockerInfo( dockerfile_info=DockerfileInfo( image_tag="datadumper", - dockerfile_directory="DataDumper" + dockerfile_directory=str(project_path( + "local_database", + "DataDumper" + )) ), volume_info=VolumeInfo( - host_path="./DataDumper/dump", + host_path=str(project_path( + "local_database", + "DataDumper", + "dump" + )), container_path="/dump" ), name="datadumper", @@ -60,10 +67,17 @@ def get_source_collector_data_dumper_info() -> DockerInfo: return DockerInfo( dockerfile_info=DockerfileInfo( image_tag="datadumper", - dockerfile_directory="DataDumper" + dockerfile_directory=str(project_path( + "local_database", + "DataDumper" + )) ), volume_info=VolumeInfo( - host_path="./DataDumper/dump", + host_path=str(project_path( + "local_database", + "DataDumper", + "dump" + )), container_path="/dump" ), name="datadumper", diff --git a/local_database/classes/DockerClient.py b/local_database/classes/DockerClient.py index bb452748..bfcc49df 100644 --- a/local_database/classes/DockerClient.py +++ b/local_database/classes/DockerClient.py @@ -2,7 +2,6 @@ from docker.errors import NotFound, APIError from local_database.DTOs import DockerfileInfo, DockerInfo -from local_database.local_db_util import get_absolute_path class DockerClient: @@ -26,42 +25,50 @@ def start_network(self, network_name): self.client.networks.create(network_name, driver="bridge") except APIError as e: # Assume already exists - print(e) + if e.response.status_code != 409: + raise e + print("Network already exists") return self.client.networks.get(network_name) def stop_network(self, network_name): self.client.networks.get(network_name).remove() - def get_image(self, dockerfile_info: DockerfileInfo): + def get_image( + self, + dockerfile_info: DockerfileInfo, + force_rebuild: bool = False + ): if dockerfile_info.dockerfile_directory: # Build image from Dockerfile self.client.images.build( - path=get_absolute_path(dockerfile_info.dockerfile_directory), - tag=dockerfile_info.image_tag + path=dockerfile_info.dockerfile_directory, + tag=dockerfile_info.image_tag, + nocache=force_rebuild, + rm=True # Remove intermediate images ) - else: - # Pull or use existing image + return + + if force_rebuild: + # Even if not from Dockerfile, re-pull to ensure freshness self.client.images.pull(dockerfile_info.image_tag) + return - def run_container( - self, - docker_info: DockerInfo, - network_name: str - ): - print(f"Running container {docker_info.name}") try: - container = self.client.containers.get(docker_info.name) - if container.status == 'running': - print(f"Container '{docker_info.name}' is already running") - return container - print("Restarting container...") - container.start() - return container + self.client.images.get(dockerfile_info.image_tag) + except NotFound: + self.client.images.pull(dockerfile_info.image_tag) + + def get_existing_container(self, docker_info_name: str): + try: + return self.client.containers.get(docker_info_name) except NotFound: - # Container does not exist; proceed to build/pull image and run - pass + return None - self.get_image(docker_info.dockerfile_info) + def create_container(self, docker_info: DockerInfo, network_name: str, force_rebuild: bool = False): + self.get_image( + docker_info.dockerfile_info, + force_rebuild=force_rebuild + ) container = self.client.containers.run( image=docker_info.dockerfile_info.image_tag, @@ -79,3 +86,33 @@ def run_container( healthcheck=docker_info.health_check_info.build_healthcheck() if docker_info.health_check_info is not None else None ) return container + + + def run_container( + self, + docker_info: DockerInfo, + network_name: str, + force_rebuild: bool = False + ): + print(f"Running container {docker_info.name}") + container = self.get_existing_container(docker_info.name) + if container is None: + return self.create_container( + docker_info=docker_info, + network_name=network_name, + force_rebuild=force_rebuild + ) + if force_rebuild: + print("Rebuilding container...") + container.remove(force=True) + return self.create_container( + docker_info=docker_info, + network_name=network_name, + force_rebuild=force_rebuild + ) + if container.status == 'running': + print(f"Container '{docker_info.name}' is already running") + return container + container.start() + return container + diff --git a/local_database/classes/DockerManager.py b/local_database/classes/DockerManager.py index ab43f852..ac294dc1 100644 --- a/local_database/classes/DockerManager.py +++ b/local_database/classes/DockerManager.py @@ -65,8 +65,13 @@ def get_image(self, dockerfile_info: DockerfileInfo): def run_container( self, docker_info: DockerInfo, + force_rebuild: bool = False ) -> DockerContainer: - raw_container = self.client.run_container(docker_info, self.network_name) + raw_container = self.client.run_container( + docker_info, + network_name=self.network_name, + force_rebuild=force_rebuild + ) return DockerContainer(self.client, raw_container) def get_containers(self): diff --git a/local_database/dump_data_sources_schema.py b/local_database/dump_data_sources_schema.py index 49627bc3..65079f53 100644 --- a/local_database/dump_data_sources_schema.py +++ b/local_database/dump_data_sources_schema.py @@ -6,7 +6,10 @@ def main(): docker_manager = DockerManager() data_sources_docker_info = get_data_sources_data_dumper_info() - container = docker_manager.run_container(data_sources_docker_info) + container = docker_manager.run_container( + data_sources_docker_info, + force_rebuild=True + ) try: container.run_command(DUMP_SH_DOCKER_PATH) finally: diff --git a/util/helper_functions.py b/util/helper_functions.py index 7d6c7f8d..deb6830b 100644 --- a/util/helper_functions.py +++ b/util/helper_functions.py @@ -1,10 +1,20 @@ import os from enum import Enum +from pathlib import Path from typing import Type from dotenv import load_dotenv from pydantic import BaseModel +def get_project_root(marker_files=(".project-root",)) -> Path: + current = Path(__file__).resolve() + for parent in [current] + list(current.parents): + if any((parent / marker).exists() for marker in marker_files): + return parent + raise FileNotFoundError("No project root found (missing marker files)") + +def project_path(*parts: str) -> Path: + return get_project_root().joinpath(*parts) def get_enum_values(enum: Type[Enum]): return [item.value for item in enum] From fbb329e6a6bd94cf5dea765badfc9e0a27e81ab1 Mon Sep 17 00:00:00 2001 From: Max Chis Date: Tue, 22 Apr 2025 15:04:52 -0400 Subject: [PATCH 05/10] DRAFT --- .github/workflows/test_app.yml | 21 ++++--- ENV.md | 8 +-- .../DataDumper/dump/data_sources_db_dump.sql | Bin 0 -> 183850 bytes local_database/classes/DockerClient.py | 2 +- local_database/create_database.py | 57 ++++++++++++++---- local_database/setup.py | 5 +- 6 files changed, 66 insertions(+), 27 deletions(-) create mode 100644 local_database/DataDumper/dump/data_sources_db_dump.sql diff --git a/.github/workflows/test_app.yml b/.github/workflows/test_app.yml index 28a41e29..1dfdd466 100644 --- a/.github/workflows/test_app.yml +++ b/.github/workflows/test_app.yml @@ -35,19 +35,19 @@ jobs: --health-retries 5 steps: - - name: Set up FDW - run: | - ./local_database/setup_fdw.sh - env: - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - name: Checkout repository uses: actions/checkout@v4 + + - name: Install PostgreSQL client tools + run: | + apt-get update + apt-get install -y postgresql-client + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt + python -m local_database.create_database --use-shell - name: Run tests run: | pytest tests/test_automated @@ -55,8 +55,13 @@ jobs: env: POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres - POSTGRES_DB: postgres + POSTGRES_DB: source_collector_test_db POSTGRES_HOST: postgres POSTGRES_PORT: 5432 + DATA_SOURCES_HOST: postgres + DATA_SOURCES_PORT: 5432 + DATA_SOURCES_USER: postgres + DATA_SOURCES_PASSWORD: postgres + DATA_SOURCES_DB: test_data_sources_db GOOGLE_API_KEY: TEST GOOGLE_CSE_ID: TEST diff --git a/ENV.md b/ENV.md index 3452ef7c..f145e20e 100644 --- a/ENV.md +++ b/ENV.md @@ -26,9 +26,9 @@ Please ensure these are properly defined in a `.env` file in the root directory. ## Data Dumper ``` -PROD_DATA_SOURCES_HOST=pdap-production-v2-do-user-8463429-0.k.db.ondigitalocean.com # The host of the production Data Sources Database -PROD_DATA_SOURCES_PORT=25060 # The port of the production Data Sources Database +PROD_DATA_SOURCES_HOST=127.0.0.1 # The host of the production Data Sources Database +PROD_DATA_SOURCES_PORT=1234 # The port of the production Data Sources Database PROD_DATA_SOURCES_USER=dump_user # The username for the production Data Sources Database -PROD_DATA_SOURCES_PASSWORD=GeriatricGeronimoGentrification # The password for the production Data Sources Database -PROD_DATA_SOURCES_DB=pdap_prod_v2_db # The database name for the production Data Sources Database +PROD_DATA_SOURCES_PASSWORD=password # The password for the production Data Sources Database +PROD_DATA_SOURCES_DB=db_name # The database name for the production Data Sources Database ``` \ No newline at end of file diff --git a/local_database/DataDumper/dump/data_sources_db_dump.sql b/local_database/DataDumper/dump/data_sources_db_dump.sql new file mode 100644 index 0000000000000000000000000000000000000000..aa27b60a2bd866ef4228dfc2af49a72617b141a3 GIT binary patch literal 183850 zcmeFa37BNrRUQ~ZLZAkTMF_D8UbkAREU7w`5xJMtg6zzw>XfoFOPN_+Y6%gFihP+F zs>p~`ELELC0)fC__FXUxi($an*y90@hw)%+gIUG{9%jtgX7d6zn0? z=|AtG|M1U8;J@cb&GW{2zjxHQS8E*IseCY=RhQ?hwMwFXOXjxHJMCsS;pdN4=g&`$ zPO7zqS644reuLWjrWfNs{6l{&RK$Nj6u;MZH}|%-cdu8hKR$@Q{bc*@2K59Nghuw-8#IscImYbF&)wWa8Hk$ok7Z9aT&YQ#2OIMOh_&Azpp4d33SGe1si6@);^^L=NqFyA9#3i@4 z51&j9>o;!h&}WY)!}RRD(;TJ?iIN^I6Uolr=EhFFadUs~sqL-$K3;U%cUtl>eU^3` zw+`kZK#!3GRX&A!w7+*V+IkqB%7>ubQLlLf-bdvulQ_a>aba~`;8UwDtgmnl;MA~x za#nc(-q+uJSl>O^-rE(xycUlnF#1iheJ$DDJ51_t-aa@yAU;!RfzQF_ll2=L$@%C` zr`-Z!ABXZxSXNqoAErp?1RC6-A@jF1tIYxYQD(PQcbKrvy&E^`yN5i`D=Cp89wd9a z=)ELVl}NS^l1sz%nPD<$pS3&9etUS4oTbgdsGpu;=qDI$NxOTLK9dL;CM|HMIY^I^ zUN^zGJLxyi26(jK0U?g-+v7IcWyg{P+OB1+xL%2I{o z5}Q!x(N3?`?5IC5lMgkokJF>Hk9W-PSD-bY z-(Fi&1M|HxU~0A16(Q}S(E!&!KNk@y;6l;vwqOK+O2Sux&}rUJQZi&(tZvm~#femp z4FK66s@J3L-7dyX-ss3WJ~d1yF_@~$>q~5FG|yqA-fMPXi8Y6#fg?Qcia=!-a^Vtn zW&rflcciUh_V;<(C6iP^(oNH&LDD|LoYg*V!v-UeS(+XRN|W9A)EJ#dFNV)EZp)m@$$ZWy@P{k2u!spBu!E)R#ux2v9- z{#VE7uP#)@0#8^CA>zLVPL;qQG77kqZ?y+-d9_Y4HIa=%PC7;SM!$~~1GV4Xu* z+#}JGi_hz6PcEm`lg)nnEKP1SY>jlJP%+E*WLbh~7Ty)wDc>kdyl7w*UVsZfNv z9~zgPUWUj4d@Os%$1rQtK595=!>y@%hZpE{KiTfy>vitIggby@8k&G^!MeQAR3&E3 zf)E;psI%SD-4I+EpfR?VQ2B9B(ZndVQzCs6RP~#!4B&%d?_5`MBRy(2lP5+U7#Hn~ zafkh8yGzV@W%6X3+_QG=X@Ah`cGC{pCdtsXQCnz=KSt3FUdALhU^}ahK_b}7cR}84 zb~=LuE2-ol?cW1fs=!7oT;SFqO?t=4HClUlm65Ro0$7SjYI@zHQA;cFr&4%bI%#rq z)E|u688A?8{b8~R_n0Q=VAQ{tUg$rswbRZKBWePAyV=FcCb`L|=x=wxE$X2$3F~*8 z&3dbYb>n8n@U<~;IHC?pyFV$lVx1pu?<1EYcLU`RsZGM zvPyvO!>@AGZ(?PvJW9gz3q)q#gg`3 zKSzz*_)N`^Zx6Z{=^=ZO?ACm*@RyM$&wPxg=gBV0?%{`^`=G3`Tp1)#d&yB7TB?sf zX*1>oDgt8;zhU8z-c&(lk?Mx$hFgsG-G zgmM&hz*RvmOi{`GUjHur))kd#!E&lWzfd~uCW8_CKaZjAE|Qj9QXR3E9M6v1(1Nl* z`x1ssCS!H#FNi9j)=E)HatBK}KqCoB`=W!UJSu^9>IGPE&7bxOeJQ`Rj6#3W zJ+_gl_GWTFy_3jFB!+5CI8|!$X%~wI%@u6z+)V)i>Dx}Pd-B*Y?VmvkdVNR)(Q!N) zVhaJ?XHjXTe)lD17&cC8K?07US;2aU3Yt7)BY~QM7KOQx9QS)?A-MFQ#^i@V?Kjj{ zYvwzUJ6je^sCnr7rG1V+x(aA2hwHu0vWCYv&)bwok_JV)II` zy0Eguo^Fh3`Sb3Wx~{IQtyf-xuO#-3QD688Ltd%(pq}^c8MG+)wYE}C`1Xb-PpaYhwwczkIZXJ7f z1sd0?A*gue7bHt0XD1kmT_r_QuS(z{syone_zX&z26}o|3_zC^&WC4Yz=|&o?%<=1 zym16)YJyP z^#{Epo~dEr{>6MyFX>}KU;SNuth&Gtw6Xa5#C>v2Xy;zHBbVO6Ef=5osYlbFXe~w1 zS8Hm6k>VDX+4~?vC17wsi8{Wlw}ipAinur*!{KlQ4K(AnVuE zAUr|gE&L-?1DxC((p0Nf*K4edVXKgDE^Rc=+l{;F#oT3%4Y_t}ck@sO@4OFHNV7$~ zP@xEZ`DDL-cx!+60C7#v3^@p4XJhyJt&QslU+A2l4Bpw{hlpEv^wEc&s9)dS<-bL# z1obuG-GzDA7>(LTbBI-_?{2;Jp+_G@(1Lc57x!Xn=&nbhTdFN^I1)TFka@(T(RO@8 z`BZ}dzKTp05f(tdd)AA1aDF?9`AN*5M}>KWTOw26L6I*e&mNTv*Z zB&Z`^g+`*D7S0N^f3hC7}h(B3;nHZ_H9Qmyl$gl1L{X&*7sW}l^Me2E?c&m-51Y?C- zp>RAY!5Hlg>2qk%%0H;EejMNDD=fd7aphO*epAnoQ>7tY`&KuTIpY=NoRy8zHKtB+ zSR!VQg#wQF)J)BguZ}UKwp6PkXk0TyT9ejJcSGz$W0}@gkvBm;tz%x#=Tqt0$7LVO zzvf{=KNQak?XH{#(s-ujCVYFz;gfYS)$_ObkI5kWi7BidPR z`z0IrD%4UJ(~kB}p?R33SFhULL8!R)NV@MXiy4fX-P=!?K>D42M0ln6JxO$jhbgWVly0&+xq|0jLaz>fQ5^pFf@RS(&a+%C!F%ws?{LW%a)F@S` zC2F`@-J!Re8wX+@)1S&;Fq`a#I1O1S;4_OGeI-2S`s)KJ6K@Po$El{uBqMpZ^simT(CKu}VHI*#+|Rp)J{z=+9EVs<0+(9>O7 zc<3$tC@tjFok#`dJ}zB8x97T+%l<^+X(h=UB@Dhu29%O4qs%@*YNB=0r7hadIpA=w zEO_tgRd=+CY79G2RW#nvyEe|ylU?`8oWaPu78`lhrRo|-4$6@ymJXT48t@am5Ow8S zBW@+*j-<@qC|*WD)5G6nt|7Sf2^J8}!d6u5h=8eSS*P1~&Kh;UiLIGa->rWYNc|Th;74yuAh$c@JybL@sz= zmGGKYP?k1Zr*lNfWgNKMyQ%BvT49&UYCrAug3`vFQM+^0=-r{9`nh}a@TOh4H;?k8 z)K<$>o-8j_!Q<+e#}Y1O^)N~5I|o@~pDI2PkhuoHmB! zu5Gli>xa0E;3BMrbD$Bm7pOpw2Ri9j!m&qisULq-h&KpKpuvpze1Ck1FKB_A{{_`C z>^1OVULE)XABMj0MjbZ{HT@*qxd`kMVVQ{zzJw={VjEJN)fu+&57tIWzjq&pCK0WX zAo7gY5~Qs=y}{5QLDs@h#(V~B2L+u05yo5ZKG8~YHivdEU%kpAXBg6|82xLSA3_e` z{X0qKG$|a}c_k5&(ruo>J=#P(4;`&x4=J@xHndL!Rj*b>mn0Ea$V9`TVx0Tk-MCS| z!mCDdv}PojCO=i7e&o;tNdZzaR)|%r#R}TC1kfk!B@{%GL$EVC>kj;G3zd|u%jGh7 zYWNi$=D&vI+Kj_eC}W&iLj6u9rMx~am7SjUGAyr<|67g?SLMl?=wmrFyo$2IWf%k= z9NcZ66VfLXeAPZq?(~MI%v#w3oenm7-o((@y|uGLTlqLBj-$jh114gY8Q61@>@JNA zqXwL9-Nh+uu7pf`LA*>O(EA`8e-Z|*KYTX215A_vqoVK<5mLeD3=^+Y z1XMsMjoP4qmHZ*_#S_6`jgUa223Ks1S5j=^WKsX7mKtiXq^wAwJsm9kE+uqDY55?^ z%pXie|~Ljb+sJUBGj&d24_B@M(<7YumeZVMzJWE!Xzz2Tvw9a87N1dt+z& zE%mMBsqOl0ME?67H)A1I&|MbhF*`f89B<`dUu92{LW=@Inx}~RMz#wiy+ABL82Rrk zfqba02ZFobkcs+IA?kkl88~A%b1+}1P_&qjx$?+HK>x0>KsPFTz^Ln%OxTylfL&KG z1KhWxxG&)_7U^yooY-vl#VVgA9Z^6~p^Y;nayxEg+7s*Lpw;se85>;1Jyz8dn?Gd0 zZfx%hzXF2sb~m4H>}(tyHf|tc-L-9~qsJ3@tfw)=Ax(Iu&(7KS3{SFKid9pQGqsj_ zW0Z4iFjHS{aL&$7(h#VVsS@PZOHgmbr>FRpQ{lv1QXv%X9vhZWTS$_&ji^jvwoxR3 z`;lc*62F}f44T1g%v$?ov>tBXs2?0|+>lOOWss3w=Wt$x(p)WKd=4&Cv-_PfcGniF z)k>1FTZG*j)cP z(!r}muWGgHD>u8~!%{0q9x2iRF3+JLLTHrobIOM5*ChuTe<7B{vY?J}&Hg3_t6u|o zz~IA`PI*VI%CQ*@8jZ7IY8m*{xD2c;a6TN_xFnFat4HBYuq%c9cj}mm_j|NmA^Jwy znJ#5VaHinAppNu#R$XbP%fjT)%t{Nr9OgG!l|e5>81kuk{b>8*CEta&hucdZC-mTjBtsX_l~G%9|i14HJ_ zg_6Z-NXj3qmdu>am92A;Ft-nnh}5V2RJkbB_cF4Jb{i9*K*>aKl>1(@e?j|7YQska z5T4?&Fl7NHuZaJ#_{PhMq#E>J*p(x7adOl{p58P;N`qE=fOvLMWd|JzzrhJ)O0U}B z6RuwGMQOd%!ZDy&r{yY)*5azW<|DhKXE^-W7A^|pEkF*c6uw0`Z8mQ09E#l04s1BN zhvWQ67EXy8uqR!njPAAUhqXXV^ukIynJF30yXeT6>V{{OJ6#eP8|myUJgwNhjse5X zYV#lIWU$j7`*tw817AI)hf-^ErkEH<$h&jVkWSv|i2+}+gODNy!l6=kTq;%R;dCD> z&@Rb1K*V}SzO)5_3F!43%0DkV!3m=%LW;F=aM1bsARF9pfSrdkzgW89X!E_qUypALFL96zb^~)-AXr7v&q2W(z*7IAnrDJv^dm598TgZZgk!?o(l zb7=k|H?jEn+d+ir5`fLUom)3{b?4_vMfy1LDc3Ati!7w%M-t9u87(?v&$?D73J}Aa zwjVRflBb-hpZF_++E=9IfULbX|krvBDGVh}`@1VQ|zO9P}t z5i<)U1erP1;nrDG|967aKOCUmluMCs%}WOU<)t9-D$+RcWCI|~FL)6mMMCWAV}AvQ zfT7vu()p8ld0bMRmAsOijk@ht`@Gq?JZr4}ZV)Sa=t>GEZeKterO=@{A8VRRG%E0W zdpJ6R71HB;bTH9;IZ@8qSyTJn<*DtS_|(3=9JR1xG_^?4@qr`v?*+*XsR7f%^6BOi z?uE+FtV099CL8V0N^!pr6(~OxO5pq)xd16hf?1RQKL*LCb(=e|1!p4auiU(6RH2C%iVtQajLU&OmS~Ul zSrh#af<%V|-n8z0o)1g$+^mDirj+0~p$lb>qw)}X3dYd9?q@07xFAwNlQHlZV+Y%8 z)>!;cK`iL516Kf?g^LfzJ4(UfR8x!y?+JoHz7!V(J%FVwTPrai zlD}CBlH5wOk)rm`)Zn1D5otu*gyO(A3~H??933)i#J)F(SQ0?YU0nJgaaR4pdBd$o z<099rq%-9=m|2Yx#ilfL3#}x`o_t_a$v+HY@@N2)&_a4rTtaneY}7Ri&@s}=W}%Tw z_xqFM8QN;qqn!pmD4$tsM0piyM3gteT%W4x7D-3R`$4d% z3LYdb|0swH-RS0N9m_}JgVHVqid0$3)1r(|!+k`VTQWJIC@vV^2^nLu(5yw{e+feH z(E$iTHrJapR(bdr*JVEksht z$g-bA1sgjE@Jhm77x7L?3~1fh+DZ_JQQt>U*cOFPDu))L$8g&l&UPWHOayPLh7vjv zV!1kDXviLk9jwER{X=mjmx#4Q$iRM`E-#}7#G~E426x~+W2Fy!TI`Q^X)u13Z@#E|4RSv)oFpFg}I!pcz z>YIW~4`u`zU4FK+%<@Orw7|s~3A~*B8Mv_4otgO`2T^*MNzOLyd=?>_ppO)1?#+;rcD1>K7p#cU zx`}&e5Gs)%M+5e0$f=8XBDnPwGZ_#rq$C}EL_5-&2b{-#Z1T+A?~?dAgW{Xt-^lm#xhmr40j2oyY|^!wX_I(5iNy4|z+Iz4h!XtZFvYA~FmS-=BT?gZ%r|OyD(3&Nc09r*;b-lx zaf`h0+PG-{Kv0K}>oX#|Y7EdFlDf}(w|CjaC1M+!cqJrUe^3hTL{(#Xzyh;m`$*C+ z=zTug6AhFDp|-%^Wx+=O9us-tkD8T{Ho9F==DV zN*dkKnIE(_w}^aHDxi)1PA^B-0@kiwwJ#7ELGjQv2Zxu(^kc1tr0C==gw2jTNt7SQ zh2Rp(hL#&yiJ}#T&X>Vk+X_PGsX|7q6S9;%K9CwjxunQ4_Nd|Pd_n0>XbCTJ$%s-a zwS@Uhdf@34yuA>j*gAS*$9&@2@i~X1G`xI{W=KuOOu1mJRvoTB66b1lo$rFpT2`Z* zLYLLx6@!{LP(_Y=^+K`8E6!8Q167-3GTeetm@rzF;082@QdQ<56==x0!M&y-dZ zI8sv9uyXXO@$_N#1y)Q`slatt1^x$gOZ@mMaD_KRI7vDdjqbpIb!_0%#R;?k6QipwI`gh*H zRfm0ZzEj(FSwG>l3hp{kd^0EQSM~#KxlnXVVDL+{d~waVQd*Dyv!JME;l-MjUi9P_ zqqUfOLX~189~YWp-2_ca>qG%}lY@)ZS7uQ0iM}ZIN|NzMgcCDH+IvB4n@vP<&5gPK z=Rr`6h+@_Ac@kBBiTSxUH1^8KDl1k&DZ)c>LF2Z8&`oiVA}xjvhS)G^Vo!JO+V;(Z z%ZRQ-h-KVW!IzLsxqkEz`N7hNgk!9)l|qEaG%c>LQI>c!$eKZ$*7xF3^+Q3b=%Hkp z<`vY0&*pcQB8oH<(X(D;Y5j(YqkIR>o!zI^*;9wW+ZY8Y91;%k-RXxm1pvEeZ|?;Rl-P=^p6JOHY{vy&EOn@ zH-A=Q%jN3gvIrGZJd7`v>3*evJ6IJ=EdQ^!<-M_Ud*kT?x`nVzn3$^6bXm^OM>%Df zmm-zOBzq#l zha_Q+?`e9U(rZD!zpS)+9luXwvQ?*YIJuW;({+gA$7Av0dWFL1N_Hzgz`hfeL_GTA z)&1aIRdsYV=&Bm8qdtpD`mtP(|Njxx^b>F;X9wVOKy%p1H6GEP2drfAsoi7lwb!qS zi}dOur^~f^#z)W1+ua(RgvM1_#)T?YFlL#$iu9GxJ5fdD9FCu*IuhPBa zW=##2vGMT72dcsksi4x1v-uo0*D4ftRE9Niwe{_&p^Vy!$d5V2KT!Aj6X<8L&dTql z&yZ3IYjI)G*}wSlpu!rL89CaDPO_BHR{7-StEx!ZDfQH%mssLIf)w#%PrV4jU$64j z;HCh3zI;l+P-H_Xhkn%QP5DOnlve{9kOI)OKVKPjWX>c1gxnN%o~lz4m$s3uR2E1*G^z6725f`o;W$@~kZ+)Af2$|K+@!jCOhl`}m#&Q0v z59bYC(R~ugJ2Y2YUX~G>T8z-t8B~3U&7(etab_LCB#3;f@^PbsDXOCn#awD5pTcpw ziy*7UsCyT?rd^-)R#8CivV=yFx;MWaJavL%aXwvd?Rw!Cpi1E`Nm{!u9J38 zhNpA<>T>crByF$aeo)$=c^CsI-;y5qx-A=Y@ySNm%L^+@;+HBes^?LFBLO$U~cnd=R4INj!29# z!dG-WMAdPY4x3cTv_1@XqWD)qj9$v5bNH!6X)x&Y%*M&yP_cT^Z^dwS2mOaK2Fq@RvlNIGl+GIj}t;yZNyMtL;c${3rybJ~Z+ zCYYEA=zJ@JaCIgfb+oge9F5q8Ox`UiSmN9#Pb0Gylb;O2L{T9x>()6=i9ci24Vn0KmGQx#8NrPPtF6B*+n@E1fq26NBoHbsgrv7A$xOzulA4IXR{D6>8s+~9f)Q21w(09* zk$YIg78<)Mh5|~H4Qn}1-ML6^Zf)Eoy?l%`6&%)LOiike*pGRp*X^B69n#^d+q z(A0;kyb0#RV&2Q@>O!?bnWS{jX_eSOab2 z=t8m3LL@+%GY)-G`@Cs@b82T4ONM)xayX^6<)QZTLDW{uqh{$G=p`Sv`_sVYR?oN@ z$$8_0an?xv`yf(RCqznUXCJKh%?vE5|4GN)@)CK@iMN;4+QuBJvj*;^(Rp|co5dds zg3YlpXr#}a!#$cFu@m?y+!`oOiaeIxq)HE!8$DbW(LxX5uEL!u z{aAyh3^V;k(Ji+Y)bo?}ygL$OuY_b#APL{NIw!ZabdzLjfhQbklI+y49VTzs+ujYc z(`u7u5EMLVQ!ekhIc6)aFQF&%X6>+{E?=gd3z{d{$|;r?-?m}9h5y5YG;)Bnw#Z4^ zYm@$qt&ZoSmo@j_G3xWSO4R0wYfL;89fzw63w$Y+iDJNzjc??CGij~xX}EHAKKQsJ zeF2p|h!zq|-z<6z|-}Ih(s_A1F5Eetm7lz2@LQ zfdKKtQeDHi?S|sN@RBS6&e~{E>m0 z@RP5(9EG)z!{fyYZ3>m(aa>z{8!9NFts)f4dLO*1+CVo8RaJf`eQuQ0Q_BuV{!>s- zjme1|C3OK>iB?75a~+11L2f59*Y}C>7M1{ z0NA1%3Po}T8M~<#l)UpLo{5e~&&?5nI6Fv0)#x!FaMhv;05fr9sW|WThlp(D_@)Rm zN`)@?+R$Om{5=dqf zDHikY>x>%VfiUADp1D51n4o4Cdr-Vy>?<9$r|y&Ft;5*YQijE7yn|H*szw1`e%xsy z`ZxlSD-?|$_M)P9T=Lg)ILjq+tm4QW=+TiOx!P)_gMk-#%_-=tV0Pn)4rK4$%_5nN z_f7yiU;lw2AcK>X4sagpO2@nu)9|HY7FV037I7k~L$xaire`W@eAqk@QQqo-tW#`r z*{`xDoW-Sc$!B9K-6Xotd3$ud@V_H^MD7KB1K=}u&jWkGlG^NU&|SKN z*$jyjU(A}i{~V;wxc|$d&MA?gEA9PSsX zic!ce!b9k^3k0R+RR-`8_Rm^FoNYjiB=8a4n)OKdB!g!+mU8<%Pqki@~W2 z>wb6&`&k3wfHZ3e{%a5d_b7+MQf1a+C)vyMlbf5v(kxQcvdW4j#Q2tjLLZo9{7MiO zH|)=W#aLO*02iY+2IjWDITGW<(jQ19E`wzC{gI34 zOioIV7rQ3L<3O)T19K*4LsNxj4Z^PlK}aHML-hiS_rP!|uQJSfq`DAm@4s&He?3Tk zY%n?VE?BO{@}S>uD^945`Np$n>JoN{ds2^w85mIx;j zM^r1vk_e}!(+e|KslOG(FlL1Cs4#q3K4|49V7D$Yw6G8-jlwgGXh=#FVIYjuz!00Y zWc_v!m4_peB}RTjc#qZgdc6;wn&`UI_kAXbO>K~kLh5fqiJUi#P0@yh}mJvJj~p!J|L&1 zWx=nFfss~vcrB;O%zhd7dt)KtJB@uL#t{wAIZeT@`^smXXrFaJB;@2fbG`ZdK|qp- zhA_S5J`}~L`jkm))+5G=7Xci?V1`LH-Szz+1aXMDbBrAUABIusv0+r>Y76-ar^=x* z1s3*#Wwv*6drOo*4B`>fo!Rz>563%7!;#g(Xk_Low1mi|$mD&`94^bhpt5>9a}?p`yAOt__j#$aJe?!xQMWIwQ(_B2(CsEYa@g ztkL?jAX+g&qFywNMT^*D^)dRQ(inwn6v!MMwPiLK6CG2Q%G(cuOjEb1aDSoKWXLSGC`~m+hNPo<# z%Yq+#@hHCl%xYoU;c%v8f>Mzf5-^v}pUhu@8oB!j{ww(Z$mLl}z+VLsi0yK6N<|_0i>0 zH*VCC^$>>!Wd$lukLG&?ky($BoqoT0K@y{s2epti?;)uL-*p>UKhJtBeNYgISbzt` zBl$w|_RP>Br5^fGq-rcR{hx>24VM2W71451VX5A^Z*!qRFZ zRA1o?LoxUmOFau09##%`vzDko0A1qedHAQQ517`f)ur{y$K#o?3>h@kaerFwyC;Qy z8Xp%$=vx)Y^Fea+=9rf&-BvVCo`G#ODbCu7#%RCpi>qj~r&SG0Z8Y?gwvdE4 zAb~5T??r`<|eau1YzXb_(&@Cl4_i?VrF=6uAyoU18eDdkrzM!ptp zgouS_OgJ$_pK#-&SeG2Zu)4UC+)vZHor`IgB_4J!2x51o40gsi^kIANWUv`^INRC#%2vTXUf(&HSlQJwW~K>8S_`&p+CJ$fchd`)t;dM;?Y2mZv3kpH z7&ArAaMG&f(oC)z^8kGjt%~`_ot#&fRx31E4Fc`S**w3GNv$zR-^sT$tFzrAh0qJo zKr9=CTnJhMXlI5n8mD=4CRrjvtp;d$mSv6bIQRRKZ(_!`%&OWtpLH}`5Z+nGZ$;^l zC1sW=I$Rw-1j#ovXv;>JN^p{GR7(pK%R4Umi7qdZM zZ@i0;j3h%eOR{R%qWdM_)6mXzcA}8WLwrh6X5=d#^jd9Th7(Pv=~==~0C*3ZxX&5t zma&{Zgm=l!L-N8P4#sNNBnM}AeJpZUyH06ORYtF#76m!+{IqkgOY)0?BpdD?tJ5U# zKGC_c%3gUU+dR)>P)7;(#a6-WuWInP{SiUZjpz`o>&6K75pafpZED1-VIZ#MOpU5Z zi~*Rg223~;)4`?Blpb`}^6-&kfnq2v9~8RM$v)lakVO#MjF0I89g0rV?B{!hQs85;-lMiiHRorN;#M_F8s z*iqCk?C=)-IOMp}FI5zf9aN%U7X?4)4_Owym;(E?loz>ut=~nqef4{}V}W>zbf6z* zn}Pa~@~GLVEC=(3E`%v0KJO33IZxcCXTGMnCaW(y3Z1n}TLtt1XL^j!QB% zr21Dq|NsAWPk}Ejd&j}mDe#)S3d}r2?zjypW0UytR*is~wxg{HA-XvHclfekacHjx zXT1g#J$sfd^}N8<=SAUOTwkud03ddHu$GV`#9Qb66^e$|dAdNh$P@VroRu5zOG~Gb zDkD;)*tsKU|A#lagSGGl;k^1YFB7EtI>=f>t3va+ac!Pza?pTBsdZOhoi{hf9ebQ| z$)n*cxrHjgeY~7(>~1A=j;lhf`Otho8{J~~J=DMV@i ztL6v6_4Awf!^n<+S(J(QQ;ZXlDV)@r$|+X@9(^wk(nl)*k3JegK5KKMLoy0^{83=U zfrvU3o)B^O9M(JPL$#5?+UE+`KI#s!p+T=nu}r(A;gSC_LGoz}(<5IgL?42Ka!ghg zc)b>OU@OX3)*eY}KU{vxj4T4NU~Ae=_Wa zyMI5j$oC7HOZ?6`)FB=u=LqL9YS)XT&XN4!$g2q7cxxHmyQFh27BXhOd*khqSisDo zem&kEM|f%^O)&px3U`qu6?!sIq~cJ$>5&jvKILCJ9FKYcuQO4%<4 zBAdB3)w36pvH~(TAVngAYZd5RtG|Ty#zMfBE4P4NS&(-kWdAC&PQ7rn6JLEfV+XPhZQnV){(; z?7Wj+gi+R^?c<=GvMc z8dcXT)f$IAAGHTKoX{G|Eo5?!4)`vE8yko9{q2pN?YGpol1zEOKU*w%U#O&$p*l@f zzO{XDxV?)jVERMcS2fr1{yS&72Ie!z?d7Xi`40Nz(&=z`KDhdt*I-3NPcB~(kL)M+ z@86#{KF}Z2EJEj+3}3JrbDXtDA4)%IC-QO7i)q&8xYzVN6o|6GyW3CJ_v=ZFeiCoB zroK#HZy!F12g%mP;RZColF|TUamIJ3p&Ib16_ zSC?4ez(&5_v;hw)jRnt7@ll!RS<5}Ip!8Std-ut0;NG++9G?{G7=J6fAiNloQ`R%Z z1yPxLg=@o+h3goXru%t4LwnwIqh0wc47>!YymF^(TmH)3!(>7(3k|=z?r`(tf|~uM z=!)Qs*Sv544)L+E_f^1AUgu%u4yA6M@oe(JGu0ZM31G=d1hABXf}+DjBJ;A1x?85;u}F-k?j;GsMtR3R(^A_SVeNdsz@Y zUP|kP|D56dQ?LdX(J?j*#_5)CCX`!w zc%%7~gYsC2Z_W)AZv#Ey8z}N{_U@eUK9*DRR12>q_wdi<*uo3pFl!YulxXxdIA8(3 zJHuD@!Fb2UkZ_iLG5!p0&V`F-TXkWxS4K!g&0@6x4w=SmZjwIs&CLigzBHjUe)5+V zj;%SD1LbOk4sl0X2Mv)i`*kQ38MFCyJ@`?(jIISoZGOjmq|1%lb$6ZiDZz1D0_B{M z+XPMFk(*DHH+CbR!|yxQ=v{a1`u~OwiywEASgbAZ74_T`(@PL0KbvSahYJ55C#*UKEY3 zDGMrN%VmQNf4Nz#BCKh%m^iI27!aY>{j`Ii0?wh^#A1q5oJ_mETz2=z9}WU!+%sQ5 zRB!{CMxz0ey{;;*kbyGRobY#!hP}t&L+q!N-4Sq2x8vra@JJ8^B%B&}pjLaCEkBQBW)iuQ3*1+omA_B)+0!MUSf@T6ySOs-s%L-29M2eW9 zE=d_pdr+sK?Eqb~HnLfL0^thlwXwbRo}m9F)x|&Fv{PN@{hcD3qZZFMquM-+$KcJt zp30;Z_24a^$I->OsF298MHDYKt!1?A z9)|hYxVk~OC@t?8B(;oiWXV0N|F!r&y10qdk=aDSFhW%-WbPO?Q8p#06}aA5;ZePS zo<){9ss)EOU;a=9e#TnsJQ|ce^5Pg4QKSR@-1aRfXUz1zeHW>TC$k+aFyRQr&OHZ> zl5d9OOxrqF5+s&GR9$<($L-IW-HXRy2hhLx2F4MEnx3}Lk<%<4;`q48WIDXxOUOtV zPJcFSb!T&s$luc`~#amXWKp3_THTxX5kr66*&lqw@@ ztgtf`T#NI?`8#Ir__EK2d?8ar!C;+p6@DUnu5z*7BvR2s8X6)leH zw#`(}G~L*_z47z`#R6<^A!FJ0;nQ*s(vH>L4P>}i5%A=gi!@}~QZsb+PHBgFYT=Wj ztXaohfsX=UW)05yP$1qQrTi;MHOgu6FkR3wZm4&6vi}a0i%#|hbrnqdcIl{d)9zyM zq&Y*mhMq_jMU+fs6VD(3Yao` zJyX!&vs2%AEPG{)WjM6An7Lbn#L~vNr)s{5WIg6?jaQRtnegU`U9w2cP|v`C@cJUh z>1wxvzi@952O`ZzySyR?>{{kzi1#8sG>V)Ig>`2grVYbQft zM>t|m+aG(kcM+ud#M5HSu)xVsSa;aT5HB{ZlVRO;GJHr(M61iywaOq_(JrHy;aG}7!7VZVrkLb{+bqG(^86g#yUIDyb1;dU1{8J`vzo(VR$weY z@pQ6PzqWB}=WwirdAKdv?`PTmkSDGm6y?q0TCMUBASS86sn)>i`(KklhD>5zqF~me zSjKWu?c%6RX#Nv3q;qy(^tVCvyC?YRnLP86Jp5Uom0Wtol~3Do*e@00Xc`VV9hFuy|GC7y5{X!jBZ-LM&*mk+!#a7F(H9u|0u*cA9kX|yc zeTM2W?Gt@el4xB83;Xt#qFX$t{iQ4?vBpBCaho-(#fCjP&PKF(8fkH@%4@5P*YQd1 zCs95$sp)Z0Fo%ulSk~1(JEtp8zJ7!7Pi?$A4hGr{WKu%Wj zyzKy&b|bD=Z{do0M!e*)#-|2(ddIrWS=6!1_ho3%pGS0aGnqze!SVJHXj#k(x;1}{ zy8R_l^-10thh#moTpJ;g z4gV#lmj^=Pa*)IqbC3DFS4LuJ!T9!aq+!g`WNECB!<^)_HbR8qt%BQh)*Sudr~$sT zvRa{Rz%*@eQHspJ_ZJ3swz!pSfEa#+@6*&bA_Fpg|5(}0smh-Vqri~(EJ;WzC; zPrTr{mqIaEQn14*Cu#cA`Qn&DIkGS4Pw_=IQ|emeqYI-8##dk`>e1gQlCtiL4+IhF zt?sFKDf_A3CivOhIH7$fnoaAF3K|_IG)mRHnVYsE z;d(`?F@-r+H?7lVA9t7H%<#QtpOTK2YU&^_XNYVmLo^eJ!UpR*2X#(Qme;d7a;AZY zIOu=gX$m$|uSK%RxjD5JuAmnb?bNRwCT5Zq-9M3GCF7K0J#s|qZy@@V_k%g>)ug1u ztw@UrYEs|iR+MqwtxCS(U1Q`s5i&$Fxm9j6|g{?xlnW&1y=UaGDNHB)s@lC1`{#lt|ByOf8euQ!}z0A+~^ zY7p`yq4hhMabX)Pc)j2^wkA0|PKd~Kdk?E_oC;+#5kIa*ScNl}RtbcV$@2!utPmnc zT>lBaj)-fdf{?TtpF$Q{WX(sG+)7!VMr3srkXG7q@#fAP`e!bzYe8X6&@C;j1{MR8 z6(2(2izO-F99wyB_JE8CS5_nSln_-asZXJ)<7t@V`0fuDSApr2lIm`iesWAwYYR)& z$_qiQyrs9ndvvZZ=NAZ$z|MnJs9^JfQz(RdJm-MnJ_X1ewNDU_9{S`Sc0_<2!qFSU z-d&1A_gzI<-uk?I`t*)dw|9T;vi3Hy1BvvS??AF>^o15>@I&Lj^qI-^prF!T-Zjshlz$G8RP|(gNwm1 zJ-dReskqmUbP?Q~?cNAq@{Kh~zlY3xy!#|#{|p$Xb;B&Vc?&-+h|^0L0N$@KC^Es%}jCylEUsdgNcgZoz6val6KQRC6|~^jd{dx1c^6N zgqi)%e&-S3OHFR9Lsoh)pZzYz5?LcMotn8nm3_t%F=N`F2+~f`L)x~~f^Oa(=F|NS zd$g$HBb5e@ZjWS<`Q|;S zH-q9<3^8eJ-D5kd{}8W)?Vvv>1GN{-Z!iWLPEss(AD&Mlz=B+oY&S&g6Jmx^bsc$= zDAK_?Mc^OY`3Zay@)bD6vgzB2;>Y(31v1eW$*8$Qq39iMo1KiXf>oxj8SME9t`vDUd=sy|;!ynqRP+3WeG4YHp>A3cPJ& zkzg7Hs6L94G<@=Rb;J{zzhPMqjtNr3+>clX{IT)H_Sh&x1&(WD+FW+avpgTM*e3v(ND%CY2b(|mkvELGQ$bcmu~U6`W#lab}EQ@&tSOm&}9!X*Q@?Md=!}}?Sk1{sT7i(yPj=uZ=i>=)th!J zjdEV!>Zk{6+30oOAJZBArG4Mmg2R~{#CnReY4-;*`v$W0TYjf`2+M%VOxck(KcaBu z7UsMG{A*GliYc$xKWTQ`@8bQ2CLFFf4X<&#hn4cEf8N8@BvVGFExGXp*Mra)v4$Q; zg3H_7#Z9ORhRE=thXeCybt6ivoB$WI2H?pc0Ho^W3eE35Nxm;3=UgPRg%GZULD)rY zDx}?^^?}~m?n=UT5Qs-uWyLREqJ{Tr(X8Dw=BINYC^5RgrX&w>;osOh^lvYCyk}7Lx>9AfKNjL0Mt}aYcalsQWl&q*vyB0)HCjFKOwnS%*=IMfP z>Kvjzv4RL*5N@`mVVzi#or}Q{+aM%dJ$E(j?+F!x|9V=aw#IiMTK$Wf&Ahk}`xTMV zDwr=Sc<{b6ItCu}odw959BgMv|Hh!&pty!Wt9jWk-iRW(8p8xLrY!iIsr7_w_WvTf zCVp&z!GWmiLgmBp3Y+ukP=u*?No~Ft@52^^>}~HZ1mD zhf{i-m4L=T>&IYPkUs` z6NQ^WB&>yTHoWJh{_;_{R~iMS0%@BwmJpuS!}Oi00mS|SFO>E0n6+5^XHX%2{0V49 zS}YkF@g+~L#o|@)0kv4X-uSakxrL4=S#JtTmatg7RwuPsyov%AOZLsQ7K?Xl=YLW1 z;+VK(v8*ypGW+M9<`6kR>9CFKsQg(}5i(o6g3@-8HO2Pk2B*FLaEhQDVW%JvW!R<= z3pRb&&M9O51mb?f{!foMU+S`PSlY(IsQ#Z`E@<`LA?xWgUgWo zj`>KJ9N+F{_I_}DQ^-hu2l9q`6Q#q8ll(fpQSW^|uNW=qXQIcxlA`7YO#O0>$Sv`b zMfQZULU~J;7iBID%hPU_k;9&|N=0|%Wr65>E^WFVX@5ctb$W$y&M|8dy@6d#drw4` zY4tjzv+iVt0$t5n4w-=X0xWni)^g~-=J0KuJ)CBf4T*k4N#uAqW3@bZ^$_)N#=7L= zT23)6y6(P%pkg2oXRPTwl@RxE#%hZx17hM}Ub1p3HL&Qqx_%`tPK&~GAv=qEutuqC z0?@Ij920=cNHrj<35*^wnw6GQu5)MLX1zenB}XFI13P^tZH>xK`=UjPV?eN~^lr)q z&W(#mMvrUg*Sggr)DSi%v;+lCzGR(j^@4(P| zP)W=)VwU6;-$o*MNhgNPgV#;IrdxcTFbm=J71^GeeR!UFxc&k$5nb}4B!kf%ijTzc z%xQN?7hMP3uPRJ@a~7xR=^TzplFDo!b=-axVH0-T#_M_Tnz4#*TNBFRoTzgjcd*_&r9Q zp=T=n2)@YCk+PK3lHxC{&-2w_^!=#c2_0!zSX#?YH45#An|eB-w2#Zx+GVL9yY2IH z@_(v`5DF;w9lkd8zPx4Fm0k}0x+sJHdXO*G_{6{6D6Xg==;LmUuR&G$<5{x7uIa&# z=tXoaZ$t;XA z_bTUyVZDeSTg+-ToPXw^X0f*5I8CG>KXJ{UA4loX&~}Q+u^k<>c`F#l4C1atbc6J> zHt5iScF>aZF}&jwaCy8O^!NkZp~jcK48zhRJjsE6kz*5`R^wy)%Q7aoo2=nF9{i|& zOyO9rLq4MA#`BtM5Z(@sXWADEBaydyc@s*7M{>9>Zwx!1P3ys2bGm5h4Mo1W5kNLa?cc6-A=|d{yAQ+SFE9)J=chN zK-Z$$dwFq@vn5iPMH2#gq>~)6uDOA9Rr}i;JKJwT)+&m%7x$WcU*<;ezEJCPzeY1u zstNTX23?bbn$+*_81>tOqjt+V1kuSdk%})n^VZo43Y$8^wT`X!IUl%C9~mb$h(|Ij zp7bDaiWVC0P3B+X<+5W3j`~VpYl4hV2q#85nYlUql4X`cloU}FPmQOJ6%2HMRASVl z-a^n(Q%Bc?+==`3WUe{iJU{REka?gXT^^4om&79+PrZ6o(PV!_S^92oA3h1IB-z?H z+(7tN4a0>s*chCfOsk2YtrZ!(rT(qTWO=cF)pzSMn-%_L?NF`B2lzldsJi;((Wsc$ z7DX6XCX@>9L7tqy1P#e)rvzHMN>k7~DL6^usosLWLOLOj(M*}?xG6GIdprn%rWWuc zvu|c~Zk9=oiH;kir!ZNi*T_MMw3ujvggND@xv+_G@?pyL5CxXc9&IEJ?IJvK4&5iN zW+%LoOIPTBHgx<*JT-_dq}Btp6xXtz8O)ni4?eIgHB+2Q{VqgW|B=C~ADJldpm2Fc z4|!UU3RDeKQ%O_GSx1X%KfpM8VL~;^+D<1MRr)dKUsd2aqz^Oq^IzdF@xvd~me+Yn zzJkn`%L|Wd&n;~R`xg1QuDmsxSL-1iz@M& za*kvkO9~A1msgC#WP`^U0#655Rv(5|#o9rz$6Hi=Gd_rIy$B{zj9E0{KYwB#Fl~+y zcZR#yF^N#>i6hQOfjCu8z=9+sh)C5QArZ)m3@tQi>%2<*jbP%m%d$oH9O_$wAW_Z$ zxrAbRVAlA1R0lRtc9^Ov#*5EVC)KFLl#5P}y0-?YGjiLBZf4`kr!qGf*DYQm0F#PP z4j$(4>>@Bd(m%Z%>6vcuNq=WCgR|ms(kbkc^Aw1HFCBzwU?8iRHT&Nd#DMl-E#ZzN z{5vZqKa%KA9Y`cz=bl335M}tT+ud1{{ux2iY4^Y)T}496M4rA3yuBC%rw$f`Cy@gX z$92^!rgMNG0Bk$k`f%2CH-dE2LyK-XQTg52o&I?R~d zY*@yh^qXg#W=f{snl;I950Xs#Ko-d~-}v-?Q8B$dYZI~iG<}CCHL{Eud2V>xlG(;O z>AkcwP}l`W@)Y17gEH~sTENw{HQvRb;R*K)?ttjR(;{{9 zfX%tg{Y4>N(q1$3PlU^w&VUgnO=*A4vz`!tq;T#C&Buoey@%~1G8T^8&>U<5<9l*V zvH7OgFY2lX>ct=#@rS-k%OfpGo5GhxeS_l(bu%66wBKdW$`)BMrM`v6)))VWr7Uuv z0(RwTky2CtE=dnj;9s$_S#|eO{!B0x|8@C_G&FK$1HKx9=x#9FMcJ4MZkChQxP&OO zv6r<&&OA5S1Med|iuS^WuGyN8s34T6EWMl14C(^fWXfz_vr%6md?mikGk?v}V$RQU z6LKgVk7gt&#N}XosjyZU8B%;o3yxn{#5kBEB9e@^kV~1sh;qqJsVSxz z%x|MRYK-_;eW{I=RJYQ(5UYkk%R8cSiw8CD(9t?Vnu#>rkcYn)m^oG9?# zm{v?_Fw*d0DEMf7(^zO_44`pUbgbv5pWV?xE~_s%h0Pkh-vY7X$JSEh9a!TCZjz6I z^mPQ1*8WSB4(xPssRF;t_?uj2U!HA#7k9<_+X`4m{}Zy}e#3XQ&l`NXo~!t8qi9I+ z>jF~atH0S-A*SIDoF}*$0b8Rj`2d z;~>%WPs1%72 z84ejPFLv|45Jk;tWp8)qX|;R*ww=!8*Wt~2tZpTBzJ4vY9zlXsUXOV3`|}al5r2qNe=kJO5joU7Nh$O& zh;-K>&P3WMIu5v5u`r&mbDETRX2l)|gzIwCe_cYk?jB>QMWI|;TC7ms6AC^;bT2}W z5EP~|=fh>tMIcv?R>RVH9+K$qa)>r@E?e(7Oek4rm_u1$y%oPyvd#y?zI8DI*5$>8 z>+ZC>5@+A)O68KvKC&e(t~K}sKyZwFTUp-4DxH{dkqX0Hd&=pi99I_CA{9OemQe^L zDXql?ks}HJI$K|tV=Sw#V)~$?0%qR_4NQ5bqdN`U>@Z6CIC+p;U+=&W8)>2XCS_Eh zt}u_@Pt?UfGeVoD)2kA15HMvQN< zIwxgXq`ol6vSYLU{3SUSsqewCOI*sk2*-VzEP1ndBMHUJVUVhz^H4X7hEYBXZbyHQ12LKFq>sUJ`IaTM0$CbiG|ySfX`dA#Okr&hXXMW`bZt&5fE9kMC!0vfPTyZ{&fj^7B%ZAsks>Z)kUmb zX!_^w8k5XGTzM)@@WI&Ae^Gs7vm@TOFk2$i|G52{1$&ld=1Sa&z-GX*d$MmK#-3Vj zc_m{HlC;qRNT2rO)a0BZaWeMg)HgPJ;&qMBo_JjkX7ot*s0-cKUG`jxv8P(a8i~#n zXL||Zk@A_2f;Ojev+Ovq8|92mmoVCoJMGH}!l>ebAiE@2kV|q!>D|xs@;I`!D&Ob_ zS+J882Sxqe4t*xRehuTG?3E4!yyH3FfgkOxzgX_<$``R4)bD7-3O$z8wU~Y zI|K$07Mf@RaJgU@*VjAE+PH>WoQ2B9!F5*Ea7W31of-AxBao64c3DE2C4`#*Qls#A zkY)wPgfuJoK%`Wi;PDdpePRxjRyGisbOy=oMDh~Q=NsjDwxOIkje0C-byAx=+t91S zO6}$t<`LV_tHTCd_hS~4T$8%D=AOuXSq|*0tb1iQ$z;N@H~cpUc%JUHYOt`D)#aMu zW@$4umIHD`_nM_`d{x`CfOk>qUN1YzC5t8WlIy%r)y;dC-z`DXR z5OW`tVJY(LI3U-ZPDn*uru~_~EcLLLN+`*J*-Ee`vr|x(XpwxxR~>olc}b|M#e!N_ zSq5rZgPpp>0XwpuFV}~ez%AGFi>@hrB?340z!p};Sz@}54$&4kuMph{d&UtaQ=*+j z1op@+ShqW6!}H&U)3}smr>xiREMP^LDO(e3QYU*UAztUgyCM&1Ev#XsLC4NwTe6#P zY;5f8?A_jYVyE6XsBi3VKH1o>AKcnGJm3(>%_r-dZ%j7#b`K8sH@0^VIaa3fJ7Fv$ zjc_0k9iPvtNgl~*_>n}kIyX0Gi7rk*a!$C%la1Y{levxk{f(#JI(O+9GIJ!y#Q-A$ zyOjNAi^2+UQ!(A1F3Vj><}P)6>ca*X%lZ6mIFrqJS)Rg4_AP8Ozhx|f(BZ@@F%%_#iv&wBr{Ndz^-cPOm(K_gU8k&kr5)&B3SY3UOM! z%c@Uq?QXy67LKPQTo|~zQ$(m@4^6*TA3}7vQepd>b#^3>xa#^GH>1$4;rKa)z^cX* z^-JR%ht(boYE*M_bAS8B#{SdE8|#pJf!e2`A7VBU4)pO*qgrPMSUm>V!a6ny*U?e7 zoHSUbOHM#|(U3lW{!rRkbO11jX+MYbcuP^)j5hAbX=|cp!;%|qb7H2rTZ7kQOsTG} zRXLwS9I6mgq7wcE6TwoF8@}`#!UK(hV~-+ENDPWkpQtZoSTxmuk=wXcmq*(%9%10D zRhB@Eq|S|kQdGOo<9aA>%tRW=VNtN%ke2$49EL=ijd3H`=ETFrjZJxDoE5m+=jFtT zPI`8y-D=!Jv^AZ-b^ZqQt)RAgM;-)rok7IyPpBCiXxX0zVu)!ry-~h7A8Eg>Ndd08 z``zCi0~kwyb>7J{qRX;h;}Y%e`I{49Rz3u2_Lgh6;S`EwD+k4_xCdbNiff(}JTLI= zMhv{_B4R{HI(gwl)|~pKaoj#1aMm9L;v7Zi($hRbX)%U6gBZ$pelvR3qz>@eJfvz7 zsOHJ0>bH++w53Vmtw|?s%}oWg7l#?nS+e|eFV!z``OLGIRq0%0{kgB$@>3PZQJiKX z^8ovYF~Dj|*wQBic)BhTkXiD(INV>ABgiEy4{@Vp(WoN%tm(Vu@p7C#O*%%r;LY!k zvwCfzvg@)s7QAEiJ@QeU$zM_y%}C9K9FCNWvNz8oRfg1j-sbgYLdN+- zhjMe6goHvL3*K81A$fDrHijL#YmDpk9-qn?-!M1 zZ=~WP-bRW=ZI*k}btjjrd3~9frEVWt6j7}dzbLgr zp<$)S;5dYAPd7x?MlLuve2O&)yf)DO`8c`ZIH&u2o)F5^A2lp}Ne(Vo}#|Uc{tFZ8`X3?d!oHKft!1 zk@}37b39|_#CkI67>S;6`Uf$_;e^v7?*ZETv^v$ah^*vIGz#9(&*S5zpMz1Pkr3M= zp9aQ`m{9YUmyx=>(G&S@sxh?gdPcq=#tNKvT4m1&cf(TF{D(g6RPa8|nmik`pwcLd z;{A-WD$g@g(2X$53i>eVX!1@%ePfh$Ys=LNB}?G$XM-fbx5!I9vdDF57NfsoE|tGm zG}I!EMVT4-zNm9rajNp@G?HwZSu(^Recyx~4zpdg=9~sB>g}ZBuqxvtSKV0kZ;Fcr z&KA9XDzS*#)On7RJSiNT_c5D0Z%;R`1LN^>8k^i>)ZdCSY8^*uXm^P_4*SZgMg#>dDVQ~EDIfPb zo!))q7FL>H48`!fILqEuGz*0*jIt~Id6ZRoi-OJtjJPO_t z(owNZhL-7y6U^u8H?_eN*Zz{`CIZ7i)ti>FZX4P9%%U{Tbp{FMZsOk zq#ZYY351pC=bCM*k3lw#+rdc(1a?<=K*axb2Fh3F@Oq*0aZ*3Iye4rAZdAMJ)69@h zEh_n3j17_}$_Pjf_|puL6*sf^x5r>ZV)}&&WzJTq4QYZKB-URIz9S&IGHvD%YSk0M z(fS~3NDDkSO=^6vY%3n5ktQY;*)`Wu`Wk#-^}z&0wHc}^~iq+(cOBAVywx0)!x zbQ)3h;QZThIG1acg#tK{@U;zR1ay64*t?r{qf&dntUMLN8){Pvu=|o^***7XYMpU# zck3o%XC~I~iSeOIx7@#)`OrT@UPhi*jRY=c4j8h>qAbYU-!ke8Ge9>TYEGk{RU_{9 z1o&>AI2(LjZ9wGpyHJyp;_hXl`Rp9rYn4yR!#y_3<;$BUs}N|^A95;O%N5U#AWaR_Fax>%%qo zKtnGIEnSR!ML_H1NkWJ0Ct?K`lh%`b+U=?$Alo15_OxCgy+oQg&;Ike9YQ+N(8Sun zJ>>Mx9K4r#J??kYdlDDtJpY7OO*yP%#>W0QY%KxxG)9M)mue7)8CP~cl8H5Y#muOQ zfflad0^@DPRCAR%*z$YbiYLMpaG0YiNI*rWT@Q!kuU-Abg6lexMmdPCaBz{?zKtlc zHi+h@m!!22&6torS+aqcd5SjPBmoF&)^hL$jF!_7Y32>;QST2Z{Md9O67rgIPlky4+j|s+3t#O!I74yUH9?%~ot&#T&V?kvH+%ZN((jY-nOxb@ylgP7K>x zty<&7mz&_q`8z)T^A=xDB_Vj7PXby%2yEWs%c;Xd)_Fgb*m_C1-WP+mx`Z2UUWM_D ztIc2uP~tOO5T|~Hy8@ThkSB(31w&|G%)aX9sD!M(Ah!9fX{%Lr1sfQ2Ls>eYnJn*M zd)BOvb$*+ZN;JHTVw28Y`+j~D{zaT~{De6CF}U+K=`y;D;70AXmJm{$w@H^(o(n2! zo*YDH=@K137ozbVQOx&WDWgmmTy*(13>%8CTHV~dT`ARF?+=vA#bxCa~3 z=u`XI_9v;2ysaVCTny%5Ta$u~AQsjfURH~L6a{+;TSVkPwE`&B9zfulG06+-bMYB1 zJ%il%dQ=~C5LorZo6TXCRYTSj%`WRb4Vmm4;vL-qIc4W!thHLsTAh@+>oPH-m&f7< z;>=yERc^Y>Nt4`S%|ZdZ>M8(q|&lHjuh7n7f2JQ`Aabw<%MBce2UfXt_{XO zYeI&{uQ2<^afahwEuHT`9MT+M?l7*k}=(y?$uMn%(pVa$aPY zSuzKXJRW=9nwYDx>$83^%GG6DW=CrRlZ=?0nsO=o9QicJ->)ml-$V|8_LRb2L=%?8O&$~+ciSQ*9M4c1PV!ftY6=4RH*$zBXe9?aY+i|1)&I+B_D2AU;99CE<5n!O-5 z@*)OwRe89p+BM|3gmTOBKa>NqIYSWKPmnwqa zs`X$H)WX$ixPcs)tu{5({FKC6^+w8d9#iuOjHnu`@kx2HBCk{M0@2%1o-D2|a-e4> zDEha!>o9MlUl$J|tp-liU6>c>sjK#d*{XL^_&ibmoG5HHsO(p=Ccq`gSzHtp0^N#i zJcxssX9Bpjgs}{?Wc#kP>}DO3(n|}e*sSO_9zZ+L@}!_6Vpcc^-6-3;G4L1G7gs7T zk>Cr&2U)OdP+7RQ2AW}NL~bza1HZ^B)*3-a(SB+Hh=U+3w$HM-tow~Se;VmD83mEq68;A8|Z$H_u-`v^QtS3)x*Ka4; zRc7G|Hx3@ceKI@s&BLTWn8(DzV;jE`z+^wEzm54uJnN+MhJ#H%qwjEp=z=#T^iy(R zB^w9XtVIvR+h)JtynqMzWUk+u=h!X!&#>hAUZy<0bvC!Qt}Og_=yHjn5Dvx9!g(1qnp0s)Jz`8tO=^v1k!{<) z=w+_9ynI;+r6aPUMk!Rfy&NeP1-v%i`nDWlw!57n@9SY+yYYYW?j~7MH#EwBaj84c zWkvtOlgZy1|B_-_T2(j7ugNQA^}nvd|1C%1BY5fgs9~j6UBV5U6xXcRw#{SQwWCiH zcA;g4Ow-~1_Vw%aeWB1k7bT=Zlh2btxn$$oVSPW@-aV-AAJWi}{L*b*gEp*hY(AOn z@7+%7Z?13Nf`+_yYj^W-JNvrLeNa78yq=?_46TCMK+fw#R+DOXLjFBpcW z{DYB*HH-M~CH!8{G1VhRcysT@jXLgRqSBT3;s5dhAy-h33+(nma%r1!8zjx7o8H%o zJ?S0uAvRpuM48wx9!c(>rnvrqKjQ9OByAKr!fbX)0wn~S1V*$|DD5nrYu@$We~}}F ziNldLaMo!6?EE+A2BaOOt`h`xsusqV`I01Ro9}6Ok zk&v}Mu}sEJwYGyYNk>QWae}aAz4GPqiwSQbsJ>j5Av7r?n>*4-v7}SgJ}de34(k0` zTt0C#7ir0CNh!orI#wdG`8kADNH)!4F~Q858b>Zg&4iQ|%7~J*`jF{J-Ry({S;JzB zl;olp0{V`iK>AA3?U?S7FN_RuOc>pIq>j|pl@i1Q@^@KM$konD1ij4-U6s&WdF)>@epW+({WTdq?MJRAfld^ijgHlt%~oJ+0AU_0UGeus5u) zCLI@8Jn*=vN%ygriM<3; z8t+aMJ5oZZj@Uw`5umrKB!k8K!TgjXV#$r)|GnslsA5Nn&iOGBVqa?@hh+oOWhKXM z4Eya9tk?ZP@zsQ%Jt)M`j@A(~QmlSh!y+cuk%m~P>FQ8Rs3RXM%3Q&gfJ`XYNMR){ z7xx_C@5cC1TSC;?>l8njg;v11l3iei z-ik1^WF0Pl6~GfRcnP5-QgQJLmVOsMzn{b4r3y{#mY|3DD>s?Tu=fX1UwHUgxI}9# z$z7!$#u=97I>5Z8tAs*>wOkc0Yo{B!OGt_!IO}Ey|BWbTYcTI#1{ze*GgiGPBF-7! z2dVl8{@4uY&QL*RNhsgfezFB8zVQEhgHd;(6sxE?dD7vu**!@I%#Ib89l>5@HnM7S z;;|%+WADD@-_AjNnGsh`FGgJbF*f3E!Lblmg>6$VruoJ|Tve>dG2~pmnhK(Shu3X1kFlY6u62{}QD z&BEHI@?!Lef5gV2cC$m-tW;i$HyrAr#-2{ENU70IuwKWGmH#fzpYK71&O}n8p0QXJ z>C-e^J4CNNu0`5Wv>U@H^5wL=^wNj_Wpq^47MCgy6EB66QY2#p`}Cix*Zzp&HN9 zzZQi;qXk4*CC7$d^kRy+do6~%-LFiEDgSOD+O%+QhBcQ=`RC*Q=TY1jaWwCx=%|Pc zA*tox;&^{)7;mf8SXhger@)!=mw6Iy_b0=cpHC>)dWP^qOMWwkFHVuJ^3i-X0pQs4 zP@TIIz93xW5pd`JcRtL)k_zf6txxHeYPS2r2IcNvG}vEq);nq+w~;ou)$5GT#Fp5q zi+Q+n8SaYy38Al5)&7HfC;T5qpTbXR775@QGA_11Z(!#PJU-C2@Cgj}&Fn=f%l zhwZb}*Z6_iwx|+kZLJ-^PG3qXA;}%Dqr6`8wl@lUmJJcRGZPM!MMSmnT(IaS5%|+M zXOFSyJl*JV;*VmSsIFl* zkj`zJLo`vk`TV@!LwZW(0?*T2--n?YSyO^=aL}MfZkMD-s+c`%A`OK{O+n{o z$`m?W6JCY?ki(RP6&1d_4E;r6_{zpe)Gv)zKWz?seYR;4s@XWEv=J}Fr`O(?WY_G8 zJw-BXW5MvMLXKQTa$Z$?XgcP1Svu)a9>Yqct4-B&le3jaxzx{ zXN%9{+4tt~EUSh*zvh6Aaxt$9HYb@5&Iupjp(2vlNOqU>5pQlBY;J7T5#V?{QMbda z&g=8U>|Pb@Ucs>-9*mwl_=j`Yy^9g_G*18dMBtU=d7vLQ`Z6E7l>>!lLusG2nuv8#GT=T} zOXC7vO~FpNy85Ew)|?{%?x#*CYyl`e@-fa&$M0s0#RYwwl*I^u6s(Po^dF6~cyV=w zU3O{*M4SL1Wy`^IO8R2+?fY`TQB{VRrrz1D*i2F#2#rr2DJW;z?GpX)xDxKfq+F1D zKon#tr&%{+=`Y8)fLp!SD{ln}JnmJflG?5H_pOtX-(kk2b4a-s)6eH1SF)K9gG&}Q zduVgts(|KT-^?;UiEHW4pPoQZa?F--M6`R??B3bpBnDBj9OZB`Y6noLb8I3j zgKEBbl_GXh>5|EtD5sB5T5Im{q+iXEt~JgRUy8&sAC+3+{n(UXPb^%_2PyfsY{@qv z6-BS&s_DH`(lVLgn1oCqH*#_lYee|x)j9=mEFeE$axAc}|r-RJJ!j)DRK6(t3$CD{NHY^2?d1O?O} z{s0t!DCwb~L^MEvz(NQ?yaNKnpI|(G9FINU_*~yjs?}ZloEiK3W;{N&$CPR3=<<~U z$Ew_a_=$0GBl@^8wg>ksIV~$csUJ~QlMKikPtmx>fV>JGE?(_#S%WQAPpHk>mO*qR zni;gSE@wu6BL>Zv^pLX&?xEnLalq|~)U39D>7+F+y4mXMrVvP`=98Iv3X{U)29bOX z%p-R~-1L4;4Rty-AIlon$)&|KoXe-rT7gUD=dAxAmsYeDd85E1G>GGidd4KD8)&Hc zXJlssAuSex$E0qOMa`*D6Gu1`x+9^y0ML`V71>TyKR@e_OJ@3{zt+ldu$9WSRw+3kZrZ7#i1YA>43t*fZ zV!SDYwXXKc(X|29By-pFB1!S9k7tKjNMYh%BP3_Er-RcS9==2gSdM7>#=%AYczkki z^`B86G2Sy^tlp^Ea8*f&l^F66p@$0zHg$#bsZF%3(6m+TMUl1xGKzRM@)<^vjQO6@ z9xw9&{dgN38pU5Tx+3SahLw~cv7vL|Q8aL3 zIadCOAfP)fcKpTj`gsaY`lX$-NDFqRcl^NU7`c?4c4=#XfvV@TqNrM$wZ;gvsq78Z zE41za-N)H+Z82V)QSB!93xiI1tR5z4fQAtC2vRFD`smKWUaBE2I4ji###oGlA?s2Q z)MMp-Ay6+JVv0T3fj)cN2s)ij@L}?*oWXyK(4XzirW1D{m37R0rTE4`$E=)7z~y;S zY!33)XDvctG0-32M>NeON3v&um(OZoro7VMEZtk=QY9K%n|jF;#|xS!fhqfUkUWdU zim$P0kfTYSk>3ul_4jZOaSE*@X@)}0uP}6>3TDj9{q5P9F@wInVR~Rk$^_sey;Zpf z{wK(mJxT%V65ldglINAo_EZ6dqH4>_TyJ$Q5CWM3>Z?jguHXM->E$zxmbCAP4|l~3*#S4xE>PqI?Ysl;OJP)K@~8*7>@?niEjbX_1aJlX!&~)rpSFf`oo6(CO(RHhMHKgRc!>PY z$1TR_vZ2q>t5@nsn#33vc$rv?$E@SNR_Npy*@SMFvI>*;ki=wOWt9C9vx2@8J+cKW zxM_{3`oq4y+>{ZyDv}MMMDrG|2(#d*PxFmfkgMIzfZ=ed_X`yRZa@Ys_STaJ90OAB zBFeqRH{!VSNW9>hf+uA2)1~T{fMurh{Gd-iqMGG0Le3 zX_{>fQ1qMCm7_N?K(j@%ni8NuvZmF%Go}}8h)Pisq^wl4n^0^`-CDNfn_dOfkzYJY z^F>8ri0FWdO(=a+q9c3!kII&Qz);aKoBMz*#nX=FY~5weK`Z~R54jU;oP-<+QY_LC z5(h3-OL}~aZiXdA>05=R2^}&DMyI+6@)wBCdNJcoJ|f_zxp48jiuaJ;d*A4g;++~~ zMU_gd#4e)W9cEgaeF{}75R39k)h%p|L8LEie#TlYJO7H&q^zDU9ebf$g=QY~ASB;D zUnFyp62(xW8=w^)TGzOiJy80{qFn4G>p;#URfd7pMIB#3_LxQoUO7ErXTy{ z=nkK~a@-u&-m-B^F9hb7$}52qwM-oiIqF~crcf)4aTFc8gE7dNhi7=&u9+KFtsAlU zHIZiaY;?;qq0;9FvH6rH^e%VL@uABgcX%KtidgRnT{3^CXwp4FDr{r zuA71tP^=rLXj$an{0LXBHe5qEP1n1#$pykGB=KsQh9kHPGXf*(QGOd(w#0>4=)Tg$*P_-tSXmXA(a;vigjS+p^4+xG-g~XZ$|#Y zz{)4S%v&0<;%4tZb8t$yHK4LwVE$?YDn*(^Dk@?C6{^r#zYD20fn{ChoNI@`QdH}} z%8yv-28x(_vCQTECB|xR#<50FnH<8^7fzF9g`WkdkQa1`-O;EZ-)}&siDdWBzYkeVvy6@Ee5Ne70Upmek)oEDRT`YgfM?9=}NERp~Ox zmchBi?+CwAj)q`aC%b_@ik3JIdV|=1sDM8&KD~j~6|N&x?nRfbl=s$2`v4i{Wtlqo z45B-w6#oy6LYEuRaHQvsYc5nG8cH;G>WUJIAzgGxNvBTAt>fe@CL>21*IeZeMD>}$ Isu&ah2SePWi2wiq literal 0 HcmV?d00001 diff --git a/local_database/classes/DockerClient.py b/local_database/classes/DockerClient.py index bfcc49df..ca9d535b 100644 --- a/local_database/classes/DockerClient.py +++ b/local_database/classes/DockerClient.py @@ -13,7 +13,7 @@ def run_command(self, command: str, container_id: str): exec_id = self.client.api.exec_create( container_id, cmd=command, - tty=True, + tty=False, stdin=False ) output_stream = self.client.api.exec_start(exec_id=exec_id, stream=True) diff --git a/local_database/create_database.py b/local_database/create_database.py index b23cc6d2..ea345fe0 100644 --- a/local_database/create_database.py +++ b/local_database/create_database.py @@ -1,8 +1,13 @@ +import argparse import os +import subprocess + import psycopg2 from psycopg2 import sql -from local_database.constants import LOCAL_DATA_SOURCES_DB_NAME, LOCAL_SOURCE_COLLECTOR_DB_NAME +from local_database.DockerInfos import get_data_sources_data_dumper_info +from local_database.classes.DockerManager import DockerManager +from local_database.constants import LOCAL_DATA_SOURCES_DB_NAME, LOCAL_SOURCE_COLLECTOR_DB_NAME, RESTORE_SH_DOCKER_PATH # Defaults (can be overridden via environment variables) POSTGRES_HOST = os.getenv("POSTGRES_HOST", "host.docker.internal") @@ -45,17 +50,6 @@ def create_database(db_name): except Exception as e: print(f"❌ Failed to create {db_name}: {e}") -def create_database_tables(): - conn = connect(LOCAL_DATA_SOURCES_DB_NAME) - with conn.cursor() as cur: - cur.execute(""" - CREATE TABLE IF NOT EXISTS test_table ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) NOT NULL - ) - """) - conn.commit() - def main(): print("Creating databases...") create_database(LOCAL_DATA_SOURCES_DB_NAME) @@ -63,3 +57,42 @@ def main(): if __name__ == "__main__": main() + parser = argparse.ArgumentParser() + + parser.add_argument( + "--use-shell", + action="store_true", + help="Use shell to run restore script" + ) + + args = parser.parse_args() + + if args.use_shell: + subprocess.run( + [ + "bash", + "-c", + RESTORE_SH_DOCKER_PATH + ], + env={ + "RESTORE_HOST": POSTGRES_HOST, + "RESTORE_USER": POSTGRES_USER, + "RESTORE_PORT": POSTGRES_PORT, + "RESTORE_DB_NAME": LOCAL_DATA_SOURCES_DB_NAME, + "RESTORE_PASSWORD": POSTGRES_PASSWORD + } + ) + os.system(RESTORE_SH_DOCKER_PATH) + exit(0) + + docker_manager = DockerManager() + data_sources_docker_info = get_data_sources_data_dumper_info() + container = docker_manager.run_container( + data_sources_docker_info, + force_rebuild=True + ) + try: + container.run_command(RESTORE_SH_DOCKER_PATH) + finally: + container.stop() + diff --git a/local_database/setup.py b/local_database/setup.py index a720ebc2..99ff1da9 100644 --- a/local_database/setup.py +++ b/local_database/setup.py @@ -29,8 +29,9 @@ def wait_for_postgres(container_id): run_command(f"docker exec {container_id} pg_isready -U postgres", check=True) print("Postgres is ready!") return - except subprocess.CalledProcessError: - print(f"Still waiting... ({i+1}/{MAX_RETRIES})") + except subprocess.CalledProcessError as e: + print(f"Still waiting... ({i + 1}/{MAX_RETRIES}) Exit code: {e.returncode}") + print(f"Output: {e.output if hasattr(e, 'output') else 'N/A'}") time.sleep(SLEEP_SECONDS) print("Postgres did not become ready in time.") sys.exit(1) From e3c00918c6ae571b64c144cde786399114f9bd0d Mon Sep 17 00:00:00 2001 From: Max Chis Date: Tue, 22 Apr 2025 15:08:39 -0400 Subject: [PATCH 06/10] DRAFT --- .github/workflows/test_app.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test_app.yml b/.github/workflows/test_app.yml index 1dfdd466..8730d331 100644 --- a/.github/workflows/test_app.yml +++ b/.github/workflows/test_app.yml @@ -48,6 +48,17 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt python -m local_database.create_database --use-shell + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: source_collector_test_db + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + DATA_SOURCES_HOST: postgres + DATA_SOURCES_PORT: 5432 + DATA_SOURCES_USER: postgres + DATA_SOURCES_PASSWORD: postgres + DATA_SOURCES_DB: test_data_sources_db - name: Run tests run: | pytest tests/test_automated From 27ef0068d0f40089be6bd6b1e738c87ca53a1b6a Mon Sep 17 00:00:00 2001 From: Max Chis Date: Tue, 22 Apr 2025 15:13:28 -0400 Subject: [PATCH 07/10] DRAFT --- .github/workflows/test_app.yml | 56 ++++++++++------------------------ 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test_app.yml b/.github/workflows/test_app.yml index 8730d331..c3c54b83 100644 --- a/.github/workflows/test_app.yml +++ b/.github/workflows/test_app.yml @@ -1,21 +1,6 @@ -# This workflow will test the Source Collector App -# Utilizing the docker-compose file in the root directory name: Test Source Collector App -on: pull_request -#jobs: -# build: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# - name: Run docker-compose -# uses: hoverkraft-tech/compose-action@v2.0.1 -# with: -# compose-file: "docker-compose.yml" -# - name: Execute tests in the running service -# run: | -# docker ps -a && docker exec data-source-identification-app-1 pytest /app/tests/test_automated +on: pull_request jobs: container-job: @@ -34,6 +19,20 @@ jobs: --health-timeout 5s --health-retries 5 + env: # <-- Consolidated env block here + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: source_collector_test_db + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + DATA_SOURCES_HOST: postgres + DATA_SOURCES_PORT: 5432 + DATA_SOURCES_USER: postgres + DATA_SOURCES_PASSWORD: postgres + DATA_SOURCES_DB: test_data_sources_db + GOOGLE_API_KEY: TEST + GOOGLE_CSE_ID: TEST + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -48,31 +47,8 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt python -m local_database.create_database --use-shell - env: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres - POSTGRES_DB: source_collector_test_db - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 - DATA_SOURCES_HOST: postgres - DATA_SOURCES_PORT: 5432 - DATA_SOURCES_USER: postgres - DATA_SOURCES_PASSWORD: postgres - DATA_SOURCES_DB: test_data_sources_db + - name: Run tests run: | pytest tests/test_automated pytest tests/test_alembic - env: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres - POSTGRES_DB: source_collector_test_db - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 - DATA_SOURCES_HOST: postgres - DATA_SOURCES_PORT: 5432 - DATA_SOURCES_USER: postgres - DATA_SOURCES_PASSWORD: postgres - DATA_SOURCES_DB: test_data_sources_db - GOOGLE_API_KEY: TEST - GOOGLE_CSE_ID: TEST From f4d41345806031dc0775fe2f64ea30620bd93f51 Mon Sep 17 00:00:00 2001 From: Max Chis Date: Tue, 22 Apr 2025 15:16:16 -0400 Subject: [PATCH 08/10] DRAFT --- local_database/create_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_database/create_database.py b/local_database/create_database.py index ea345fe0..58b15508 100644 --- a/local_database/create_database.py +++ b/local_database/create_database.py @@ -77,7 +77,7 @@ def main(): env={ "RESTORE_HOST": POSTGRES_HOST, "RESTORE_USER": POSTGRES_USER, - "RESTORE_PORT": POSTGRES_PORT, + "RESTORE_PORT": str(POSTGRES_PORT), "RESTORE_DB_NAME": LOCAL_DATA_SOURCES_DB_NAME, "RESTORE_PASSWORD": POSTGRES_PASSWORD } From 861ea7198147ee38973f8059ca433b4e8f7c2dbe Mon Sep 17 00:00:00 2001 From: Max Chis Date: Tue, 22 Apr 2025 16:34:20 -0400 Subject: [PATCH 09/10] feat(database): begin setting up FDW - initial link --- ENV.md | 20 ++++++++++++++----- ...3f1272f94b9_set_up_foreign_data_wrapper.py | 10 +++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ENV.md b/ENV.md index f145e20e..fdd7d029 100644 --- a/ENV.md +++ b/ENV.md @@ -23,12 +23,22 @@ Please ensure these are properly defined in a `.env` file in the root directory. [^1:] The user account in question will require elevated permissions to access certain endpoints. At a minimum, the user will require the `source_collector` and `db_write` permissions. +## Foreign Data Wrapper (FDW) +``` +FDW_DATA_SOURCES_HOST=127.0.0.1 # The host of the Data Sources Database, used for FDW setup +FDW_DATA_SOURCES_PORT=1234 # The port of the Data Sources Database, used for FDW setup +FDW_DATA_SOURCES_USER=fdw_user # The username for the Data Sources Database, used for FDW setup +FDW_DATA_SOURCES_PASSWORD=password # The password for the Data Sources Database, used for FDW setup +FDW_DATA_SOURCES_DB=db_name # The database name for the Data Sources Database, used for FDW setup + +``` + ## Data Dumper ``` -PROD_DATA_SOURCES_HOST=127.0.0.1 # The host of the production Data Sources Database -PROD_DATA_SOURCES_PORT=1234 # The port of the production Data Sources Database -PROD_DATA_SOURCES_USER=dump_user # The username for the production Data Sources Database -PROD_DATA_SOURCES_PASSWORD=password # The password for the production Data Sources Database -PROD_DATA_SOURCES_DB=db_name # The database name for the production Data Sources Database +PROD_DATA_SOURCES_HOST=127.0.0.1 # The host of the production Data Sources Database, used for Data Dumper +PROD_DATA_SOURCES_PORT=1234 # The port of the production Data Sources Database, used for Data Dumper +PROD_DATA_SOURCES_USER=dump_user # The username for the production Data Sources Database, used for Data Dumper +PROD_DATA_SOURCES_PASSWORD=password # The password for the production Data Sources Database, used for Data Dumper +PROD_DATA_SOURCES_DB=db_name # The database name for the production Data Sources Database, used for Data Dumper ``` \ No newline at end of file diff --git a/alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py b/alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py index 5c1adf18..1b73f5f4 100644 --- a/alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py +++ b/alembic/versions/2025_04_21_1817-13f1272f94b9_set_up_foreign_data_wrapper.py @@ -21,11 +21,11 @@ def upgrade() -> None: load_dotenv() - remote_host = os.getenv("DATA_SOURCES_HOST") - user = os.getenv("DATA_SOURCES_USER") - password = os.getenv("DATA_SOURCES_PASSWORD") - db_name = os.getenv("DATA_SOURCES_DB") - port = os.getenv("DATA_SOURCES_PORT") + remote_host = os.getenv("FDW_DATA_SOURCES_HOST") + user = os.getenv("FDW_DATA_SOURCES_USER") + password = os.getenv("FDW_DATA_SOURCES_PASSWORD") + db_name = os.getenv("FDW_DATA_SOURCES_DB") + port = os.getenv("FDW_DATA_SOURCES_PORT") op.execute(f"CREATE EXTENSION IF NOT EXISTS postgres_fdw;") From 8e47a33fdd9b9c20cbca934f21b59bf8874567e2 Mon Sep 17 00:00:00 2001 From: Max Chis Date: Tue, 22 Apr 2025 16:36:17 -0400 Subject: [PATCH 10/10] feat(database): begin setting up FDW - initial link --- .github/workflows/test_app.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_app.yml b/.github/workflows/test_app.yml index c3c54b83..c869304a 100644 --- a/.github/workflows/test_app.yml +++ b/.github/workflows/test_app.yml @@ -30,6 +30,11 @@ jobs: DATA_SOURCES_USER: postgres DATA_SOURCES_PASSWORD: postgres DATA_SOURCES_DB: test_data_sources_db + FDW_DATA_SOURCES_HOST: postgres + FDW_DATA_SOURCES_PORT: 5432 + FDW_DATA_SOURCES_USER: postgres + FDW_DATA_SOURCES_PASSWORD: postgres + FDW_DATA_SOURCES_DB: test_data_sources_db GOOGLE_API_KEY: TEST GOOGLE_CSE_ID: TEST