From b5cdfc3d0ebb03213e9b71e79a0ad3f8d7598cbd Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 27 May 2025 11:00:29 +0200 Subject: [PATCH 1/4] fix: VO is VARCHAR(64) --- diracx-db/src/diracx/db/sql/job/schema.py | 4 ++-- diracx-db/src/diracx/db/sql/pilot_agents/schema.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index bb9f60bf1..23cce44f9 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -62,7 +62,7 @@ class Jobs(JobDBBase): job_name = Column("JobName", String(128), default="Unknown") owner = Column("Owner", String(64), default="Unknown") owner_group = Column("OwnerGroup", String(128), default="Unknown") - vo = Column("VO", String(32)) + vo = Column("VO", String(64)) submission_time = NullColumn("SubmissionTime", DateTime) reschedule_time = NullColumn("RescheduleTime", DateTime) last_update_time = NullColumn("LastUpdateTime", DateTime) @@ -153,6 +153,6 @@ class JobCommands(JobDBBase): ) command = Column("Command", String(100)) arguments = Column("Arguments", String(100)) - status = Column("Status", String(64), default="Received") + status = Column("Status", String(32), default="Received") reception_time = Column("ReceptionTime", DateTime, primary_key=True) execution_time = NullColumn("ExecutionTime", DateTime) diff --git a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py index bff7c460c..7b75c20ba 100644 --- a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py +++ b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py @@ -26,7 +26,7 @@ class PilotAgents(PilotAgentsDBBase): destination_site = Column("DestinationSite", String(128), default="NotAssigned") queue = Column("Queue", String(128), default="Unknown") grid_site = Column("GridSite", String(128), default="Unknown") - vo = Column("VO", String(128)) + vo = Column("VO", String(64)) grid_type = Column("GridType", String(32), default="LCG") benchmark = Column("BenchMark", Double, default=0.0) submission_time = NullColumn("SubmissionTime", DateTime) From ab08fe49694f058722b0418abf17e08b03dc1c74 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 2 Jun 2025 16:29:37 +0200 Subject: [PATCH 2/4] feat: define summary tables --- diracx-db/src/diracx/db/sql/job/schema.py | 127 ++++++++++++++++++ .../src/diracx/db/sql/pilot_agents/schema.py | 73 ++++++++++ 2 files changed, 200 insertions(+) diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index 23cce44f9..2e20aa3a8 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -8,6 +8,7 @@ Integer, String, Text, + UniqueConstraint, ) from sqlalchemy.orm import declarative_base @@ -156,3 +157,129 @@ class JobCommands(JobDBBase): status = Column("Status", String(32), default="Received") reception_time = Column("ReceptionTime", DateTime, primary_key=True) execution_time = NullColumn("ExecutionTime", DateTime) + + +class JobsHistorySummary(JobDBBase): + __tablename__ = "JobsHistorySummary" + id = Column("ID", Integer, primary_key=True, autoincrement=True) + status = Column("Status", String(32)) + site = Column("Site", String(100)) + owner = Column("Owner", String(64)) + owner_group = Column("OwnerGroup", String(128)) + vo = Column("VO", String(64)) + job_group = Column("JobGroup", String(32)) + job_type = Column("JobType", String(32)) + application_status = Column("ApplicationStatus", String(255)) + minor_status = Column("MinorStatus", String(128)) + job_count = Column("JobCount", Integer, default=0) + reschedule_sum = Column("RescheduleSum", Integer, default=0) + + __table_args__ = ( + UniqueConstraint( + "Status", + "Site", + "Owner", + "OwnerGroup", # TODO: OwnerGroup(32) + "VO", + "JobGroup", + "JobType", + "ApplicationStatus", # TODO: ApplicationStatus(128) + "MinorStatus", + name="uq_summary", + ), + ) + + +# TODO: + +# DELIMITER // + +# CREATE TRIGGER trg_Jobs_insert +# AFTER INSERT ON Jobs +# FOR EACH ROW +# BEGIN +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType, +# ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO, +# NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus, +# NEW.MinorStatus, 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; +# END; +# // + +# CREATE TRIGGER trg_Jobs_delete +# AFTER DELETE ON Jobs +# FOR EACH ROW +# BEGIN +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Remove zero rows +# DELETE FROM JobsHistorySummary +# WHERE JobCount = 0 +# AND Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; +# END; +# // + +# CREATE TRIGGER trg_Jobs_update_status +# AFTER UPDATE ON Jobs +# FOR EACH ROW +# BEGIN +# IF OLD.Status != NEW.Status THEN + +# -- Decrease count from old status +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Delete row if count drops to zero +# DELETE FROM JobsHistorySummary WHERE JobCount = 0; + +# -- Increase count for new status +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, JobGroup, VO, +# JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup, +# NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus, +# 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; + +# END IF; +# END; +# // + +# DELIMITER ; diff --git a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py index 7b75c20ba..fe80b9ee6 100644 --- a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py +++ b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py @@ -58,3 +58,76 @@ class PilotOutput(PilotAgentsDBBase): pilot_id = Column("PilotID", Integer, primary_key=True) std_output = Column("StdOutput", Text) std_error = Column("StdError", Text) + + +class PilotsHistorySummary(PilotAgentsDBBase): + __tablename__ = "PilotsHistorySummary" + grid_site = Column("GridSite", String(128), primary_key=True) + destination_site = Column("DestinationSite", String(128), primary_key=True) + status = Column("Status", String(32), primary_key=True) + vo = Column("VO", String(64), primary_key=True) + pilot_count = Column("PilotCount", Integer, default=0) + + +# TODO: + +# DELIMITER // + +# CREATE TRIGGER trg_PilotAgents_insert +# AFTER INSERT ON PilotAgents +# FOR EACH ROW +# BEGIN +# INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount) +# VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1) +# ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1; +# END; +# // + +# CREATE TRIGGER trg_PilotAgents_delete +# AFTER DELETE ON PilotAgents +# FOR EACH ROW +# BEGIN +# UPDATE PilotsHistorySummary +# SET PilotCount = PilotCount - 1 +# WHERE GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; + +# -- Remove zero rows +# DELETE FROM PilotsHistorySummary +# WHERE PilotCount = 0 +# AND GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; +# END; +# // + +# CREATE TRIGGER trg_PilotAgents_update_status +# AFTER UPDATE ON PilotAgents +# FOR EACH ROW +# BEGIN +# IF OLD.Status != NEW.Status THEN + +# -- Decrease count from old status +# UPDATE PilotsHistorySummary +# SET PilotCount = PilotCount - 1 +# WHERE GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; + +# -- Delete row if count drops to zero +# DELETE FROM PilotsHistorySummary WHERE PilotCount = 0; + +# -- Increase count for new status +# INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount) +# VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1) +# ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1; + +# END IF; +# END; +# // + +# DELIMITER ; From 93100ee6dabfb68c3e3ceb3a5fad267cf86090a8 Mon Sep 17 00:00:00 2001 From: AcquaDiGiorgio <64639359+AcquaDiGiorgio@users.noreply.github.com> Date: Tue, 1 Jul 2025 14:52:14 +0200 Subject: [PATCH 3/4] Alembic automatic trigger creation --- diracx-db/pyproject.toml | 1 + diracx-db/src/diracx/db/sql/job/alembic.ini | 141 +++++++++ .../src/diracx/db/sql/job/migrations/README | 1 + .../src/diracx/db/sql/job/migrations/env.py | 96 ++++++ .../db/sql/job/migrations/script.py.mako | 36 +++ .../migrations/versions/79fce7dece6a_init.py | 154 +++++++++ .../migrations/versions/b97381ed616f_new.py | 41 +++ diracx-db/src/diracx/db/sql/job/schema.py | 299 ++++++++++++------ diracx-db/src/diracx/db/sql/utils/__init__.py | 2 + diracx-db/src/diracx/db/sql/utils/alembic.py | 186 +++++++++++ 10 files changed, 862 insertions(+), 95 deletions(-) create mode 100644 diracx-db/src/diracx/db/sql/job/alembic.ini create mode 100644 diracx-db/src/diracx/db/sql/job/migrations/README create mode 100644 diracx-db/src/diracx/db/sql/job/migrations/env.py create mode 100644 diracx-db/src/diracx/db/sql/job/migrations/script.py.mako create mode 100644 diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py create mode 100644 diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py create mode 100644 diracx-db/src/diracx/db/sql/utils/alembic.py diff --git a/diracx-db/pyproject.toml b/diracx-db/pyproject.toml index 8a5e87d8a..e7c673c3c 100644 --- a/diracx-db/pyproject.toml +++ b/diracx-db/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "pydantic >=2.10", "sqlalchemy[aiomysql,aiosqlite] >= 2", "uuid-utils", + "alembic" ] dynamic = ["version"] diff --git a/diracx-db/src/diracx/db/sql/job/alembic.ini b/diracx-db/src/diracx/db/sql/job/alembic.ini new file mode 100644 index 000000000..cf1177faf --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/alembic.ini @@ -0,0 +1,141 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/diracx-db/src/diracx/db/sql/job/migrations/README b/diracx-db/src/diracx/db/sql/job/migrations/README new file mode 100644 index 000000000..e0d0858f2 --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. \ No newline at end of file diff --git a/diracx-db/src/diracx/db/sql/job/migrations/env.py b/diracx-db/src/diracx/db/sql/job/migrations/env.py new file mode 100644 index 000000000..37f70f776 --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/env.py @@ -0,0 +1,96 @@ +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from diracx.db.sql.job.schema import JobDBBase as Base +from diracx.db.sql.utils.base import BaseSQLDB + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +url = "mysql+aiomysql://user:password@localhost:3306/JobDB" +#url = BaseSQLDB.available_urls()["JobDB"] #! This Env Variable refers to a non root user, which doesn't have enough permissions +config.set_main_option("sqlalchemy.url", url) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + # The mutable_structure lets store some extra information for future rendering + template_args={"mutable_structure": {"triggers": []}} + ) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """In this scenario we need to create an Engine + and associate a connection with the context. + """ + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/diracx-db/src/diracx/db/sql/job/migrations/script.py.mako b/diracx-db/src/diracx/db/sql/job/migrations/script.py.mako new file mode 100644 index 000000000..08dc41dcc --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/script.py.mako @@ -0,0 +1,36 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import diracx +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + +% if mutable_structure['triggers']: +# Trigger definitions +from diracx.db.sql.utils.alembic import Trigger +% for trigger in mutable_structure['triggers']: +${trigger.name} = ${repr(trigger)} +% endfor +% endif + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py b/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py new file mode 100644 index 000000000..57183d966 --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py @@ -0,0 +1,154 @@ +"""init + +Revision ID: 79fce7dece6a +Revises: +Create Date: 2025-06-20 16:31:53.704617 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +import diracx + +# revision identifiers, used by Alembic. +revision: str = '79fce7dece6a' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('JobJDLs', + sa.Column('JobID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('JDL', sa.Text(), nullable=False), + sa.Column('JobRequirements', sa.Text(), nullable=False), + sa.Column('OriginalJDL', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('JobID') + ) + op.create_table('JobsHistorySummary', + sa.Column('ID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('Status', sa.String(length=32), nullable=False), + sa.Column('Site', sa.String(length=100), nullable=False), + sa.Column('Owner', sa.String(length=64), nullable=False), + sa.Column('OwnerGroup', sa.String(length=128), nullable=False), + sa.Column('VO', sa.String(length=64), nullable=False), + sa.Column('JobGroup', sa.String(length=32), nullable=False), + sa.Column('JobType', sa.String(length=32), nullable=False), + sa.Column('ApplicationStatus', sa.String(length=255), nullable=False), + sa.Column('MinorStatus', sa.String(length=128), nullable=False), + sa.Column('JobCount', sa.Integer(), nullable=False), + sa.Column('RescheduleSum', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('ID'), + sa.UniqueConstraint('Status', 'Site', 'Owner', 'OwnerGroup', 'VO', 'JobGroup', 'JobType', 'ApplicationStatus', 'MinorStatus', name='uq_summary') + ) + op.create_table('Jobs', + sa.Column('JobID', sa.Integer(), nullable=False), + sa.Column('JobType', sa.String(length=32), nullable=False), + sa.Column('JobGroup', sa.String(length=32), nullable=False), + sa.Column('Site', sa.String(length=100), nullable=False), + sa.Column('JobName', sa.String(length=128), nullable=False), + sa.Column('Owner', sa.String(length=64), nullable=False), + sa.Column('OwnerGroup', sa.String(length=128), nullable=False), + sa.Column('VO', sa.String(length=64), nullable=False), + sa.Column('SubmissionTime', sa.DateTime(), nullable=True), + sa.Column('RescheduleTime', sa.DateTime(), nullable=True), + sa.Column('LastUpdateTime', sa.DateTime(), nullable=True), + sa.Column('StartExecTime', sa.DateTime(), nullable=True), + sa.Column('HeartBeatTime', sa.DateTime(), nullable=True), + sa.Column('EndExecTime', sa.DateTime(), nullable=True), + sa.Column('Status', sa.String(length=32), nullable=False), + sa.Column('MinorStatus', sa.String(length=128), nullable=False), + sa.Column('ApplicationStatus', sa.String(length=255), nullable=False), + sa.Column('UserPriority', sa.Integer(), nullable=False), + sa.Column('RescheduleCounter', sa.Integer(), nullable=False), + sa.Column('VerifiedFlag', diracx.db.sql.utils.types.EnumBackedBool(), nullable=False), + sa.Column('AccountedFlag', diracx.db.sql.job.schema.AccountedFlagEnum(), nullable=False), + sa.ForeignKeyConstraint(['JobID'], ['JobJDLs.JobID'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('JobID') + ) + op.create_index('ApplicationStatus', 'Jobs', ['ApplicationStatus'], unique=False) + op.create_index('JobGroup', 'Jobs', ['JobGroup'], unique=False) + op.create_index('JobType', 'Jobs', ['JobType'], unique=False) + op.create_index('LastUpdateTime', 'Jobs', ['LastUpdateTime'], unique=False) + op.create_index('MinorStatus', 'Jobs', ['MinorStatus'], unique=False) + op.create_index('Owner', 'Jobs', ['Owner'], unique=False) + op.create_index('OwnerGroup', 'Jobs', ['OwnerGroup'], unique=False) + op.create_index('Site', 'Jobs', ['Site'], unique=False) + op.create_index('Status', 'Jobs', ['Status'], unique=False) + op.create_index('StatusSite', 'Jobs', ['Status', 'Site'], unique=False) + op.create_table('AtticJobParameters', + sa.Column('JobID', sa.Integer(), nullable=False), + sa.Column('Name', sa.String(length=100), nullable=False), + sa.Column('Value', sa.Text(), nullable=False), + sa.Column('RescheduleCycle', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('JobID', 'Name') + ) + op.create_table('HeartBeatLoggingInfo', + sa.Column('JobID', sa.Integer(), nullable=False), + sa.Column('Name', sa.String(length=100), nullable=False), + sa.Column('Value', sa.Text(), nullable=False), + sa.Column('HeartBeatTime', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('JobID', 'Name', 'HeartBeatTime') + ) + op.create_table('InputData', + sa.Column('JobID', sa.Integer(), nullable=False), + sa.Column('LFN', sa.String(length=255), nullable=False), + sa.Column('Status', sa.String(length=32), nullable=False), + sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('JobID', 'LFN') + ) + op.create_table('JobCommands', + sa.Column('JobID', sa.Integer(), nullable=False), + sa.Column('Command', sa.String(length=100), nullable=False), + sa.Column('Arguments', sa.String(length=100), nullable=False), + sa.Column('Status', sa.String(length=32), nullable=False), + sa.Column('ReceptionTime', sa.DateTime(), nullable=False), + sa.Column('ExecutionTime', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('JobID', 'ReceptionTime') + ) + op.create_table('JobParameters', + sa.Column('JobID', sa.Integer(), nullable=False), + sa.Column('Name', sa.String(length=100), nullable=False), + sa.Column('Value', sa.Text(), nullable=False), + sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('JobID', 'Name') + ) + op.create_table('OptimizerParameters', + sa.Column('JobID', sa.Integer(), nullable=False), + sa.Column('Name', sa.String(length=100), nullable=False), + sa.Column('Value', sa.Text(), nullable=False), + sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('JobID', 'Name') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('OptimizerParameters') + op.drop_table('JobParameters') + op.drop_table('JobCommands') + op.drop_table('InputData') + op.drop_table('HeartBeatLoggingInfo') + op.drop_table('AtticJobParameters') + op.drop_index('StatusSite', table_name='Jobs') + op.drop_index('Status', table_name='Jobs') + op.drop_index('Site', table_name='Jobs') + op.drop_index('OwnerGroup', table_name='Jobs') + op.drop_index('Owner', table_name='Jobs') + op.drop_index('MinorStatus', table_name='Jobs') + op.drop_index('LastUpdateTime', table_name='Jobs') + op.drop_index('JobType', table_name='Jobs') + op.drop_index('JobGroup', table_name='Jobs') + op.drop_index('ApplicationStatus', table_name='Jobs') + op.drop_table('Jobs') + op.drop_table('JobsHistorySummary') + op.drop_table('JobJDLs') + # ### end Alembic commands ### diff --git a/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py b/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py new file mode 100644 index 000000000..b9ac408fa --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py @@ -0,0 +1,41 @@ +"""new + +Revision ID: b97381ed616f +Revises: 79fce7dece6a +Create Date: 2025-07-01 12:41:17.490790 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import diracx +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision: str = 'b97381ed616f' +down_revision: Union[str, Sequence[str], None] = '79fce7dece6a' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +from diracx.db.sql.utils.alembic import Trigger +trg_Jobs_insert = Trigger(name='trg_Jobs_insert', when='AFTER', action='INSERT', table='Jobs', time='FOR EACH ROW', body=' INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType,\n ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO,\n NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus,\n NEW.MinorStatus, 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;') +trg_Jobs_delete = Trigger(name='trg_Jobs_delete', when='AFTER', action='DELETE', table='Jobs', time='FOR EACH ROW', body=' UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Remove zero rows\n DELETE FROM JobsHistorySummary\n WHERE JobCount = 0\n AND Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;') +trg_Jobs_update_status = Trigger(name='trg_Jobs_update_status', when='AFTER', action='UPDATE', table='Jobs', time='FOR EACH ROW', body=' IF OLD.Status != NEW.Status THEN\n\n -- Decrease count from old status\n UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Delete row if count drops to zero\n DELETE FROM JobsHistorySummary WHERE JobCount = 0;\n\n -- Increase count for new status\n INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, JobGroup, VO,\n JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup,\n NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus,\n 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;\n\n END IF;') + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_trigger(trg_Jobs_insert) + op.create_trigger(trg_Jobs_delete) + op.create_trigger(trg_Jobs_update_status) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_trigger(trg_Jobs_update_status) + op.drop_trigger(trg_Jobs_delete) + op.drop_trigger(trg_Jobs_insert) + # ### end Alembic commands ### diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index 2e20aa3a8..6784eaf17 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -12,7 +12,7 @@ ) from sqlalchemy.orm import declarative_base -from ..utils import Column, EnumBackedBool, NullColumn +from ..utils import Column, EnumBackedBool, NullColumn, Trigger JobDBBase = declarative_base() @@ -189,97 +189,206 @@ class JobsHistorySummary(JobDBBase): ), ) - -# TODO: - -# DELIMITER // - -# CREATE TRIGGER trg_Jobs_insert -# AFTER INSERT ON Jobs -# FOR EACH ROW -# BEGIN -# INSERT INTO JobsHistorySummary ( -# Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType, -# ApplicationStatus, MinorStatus, JobCount, RescheduleSum -# ) -# VALUES ( -# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO, -# NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus, -# NEW.MinorStatus, 1, NEW.RescheduleCounter -# ) -# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, -# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; -# END; -# // - -# CREATE TRIGGER trg_Jobs_delete -# AFTER DELETE ON Jobs -# FOR EACH ROW -# BEGIN -# UPDATE JobsHistorySummary -# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter -# WHERE Status = OLD.Status -# AND Site = OLD.Site -# AND Owner = OLD.Owner -# AND OwnerGroup = OLD.OwnerGroup -# AND VO = OLD.VO -# AND JobGroup = OLD.JobGroup -# AND JobType = OLD.JobType -# AND ApplicationStatus = OLD.ApplicationStatus -# AND MinorStatus = OLD.MinorStatus; - -# -- Remove zero rows -# DELETE FROM JobsHistorySummary -# WHERE JobCount = 0 -# AND Status = OLD.Status -# AND Site = OLD.Site -# AND Owner = OLD.Owner -# AND OwnerGroup = OLD.OwnerGroup -# AND VO = OLD.VO -# AND JobGroup = OLD.JobGroup -# AND JobType = OLD.JobType -# AND ApplicationStatus = OLD.ApplicationStatus -# AND MinorStatus = OLD.MinorStatus; -# END; -# // - -# CREATE TRIGGER trg_Jobs_update_status -# AFTER UPDATE ON Jobs -# FOR EACH ROW -# BEGIN -# IF OLD.Status != NEW.Status THEN - -# -- Decrease count from old status -# UPDATE JobsHistorySummary -# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter -# WHERE Status = OLD.Status -# AND Site = OLD.Site -# AND Owner = OLD.Owner -# AND OwnerGroup = OLD.OwnerGroup -# AND VO = OLD.VO -# AND JobGroup = OLD.JobGroup -# AND JobType = OLD.JobType -# AND ApplicationStatus = OLD.ApplicationStatus -# AND MinorStatus = OLD.MinorStatus; - -# -- Delete row if count drops to zero -# DELETE FROM JobsHistorySummary WHERE JobCount = 0; - -# -- Increase count for new status -# INSERT INTO JobsHistorySummary ( -# Status, Site, Owner, OwnerGroup, JobGroup, VO, -# JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum -# ) -# VALUES ( -# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup, -# NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus, -# 1, NEW.RescheduleCounter -# ) -# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, -# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; - -# END IF; -# END; -# // - -# DELIMITER ; +########################################################### +# Triggers defined for SqlAlchemy +########################################################### +# This take place during the execution on the client, +# the MySQL database does not see this, the triggers +# are executed within the python code +########################################################### + +# trg_Jobs_insert = DDL('''\ +# CREATE TRIGGER trg_Jobs_insert +# AFTER INSERT ON Jobs +# FOR EACH ROW +# BEGIN +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType, +# ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO, +# NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus, +# NEW.MinorStatus, 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; +# END;''') + +# trg_Jobs_delete = DDL('''\ +# CREATE TRIGGER trg_Jobs_delete +# AFTER DELETE ON Jobs +# FOR EACH ROW +# BEGIN +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Remove zero rows +# DELETE FROM JobsHistorySummary +# WHERE JobCount = 0 +# AND Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; +# END;''') + +# trg_Jobs_update_status = DDL('''\ +# CREATE TRIGGER trg_Jobs_update_status +# AFTER UPDATE ON Jobs +# FOR EACH ROW +# BEGIN +# IF OLD.Status != NEW.Status THEN + +# -- Decrease count from old status +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Delete row if count drops to zero +# DELETE FROM JobsHistorySummary WHERE JobCount = 0; + +# -- Increase count for new status +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, JobGroup, VO, +# JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup, +# NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus, +# 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; + +# END IF; +# END;''') + +# # Sqlalchemy trigger creation +# event.listen(Jobs.__table__, 'after_create', trg_Jobs_insert) +# event.listen(Jobs.__table__, 'after_create', trg_Jobs_delete) +# event.listen(Jobs.__table__, 'after_create', trg_Jobs_update_status) + +########################################################### +# Triggers defined for Alembic +########################################################### +# If we use Alembic, we need a class prepared for it to +# understand, using ReplaceableObjects. +# This creates the triggers at SQL level, inside the db. +########################################################### + +trg_Jobs_insert = Trigger( + name="trg_Jobs_insert", + when="AFTER", + action="INSERT", + table="Jobs", + time="FOR EACH ROW", + body="""\ + INSERT INTO JobsHistorySummary ( + Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType, + ApplicationStatus, MinorStatus, JobCount, RescheduleSum + ) + VALUES ( + NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO, + NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus, + NEW.MinorStatus, 1, NEW.RescheduleCounter + ) + ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, + RescheduleSum = RescheduleSum + NEW.RescheduleCounter;""") + +trg_Jobs_delete = Trigger( + name="trg_Jobs_delete", + when="AFTER", + action="DELETE", + table="Jobs", + time="FOR EACH ROW", + body="""\ + UPDATE JobsHistorySummary + SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter + WHERE Status = OLD.Status + AND Site = OLD.Site + AND Owner = OLD.Owner + AND OwnerGroup = OLD.OwnerGroup + AND VO = OLD.VO + AND JobGroup = OLD.JobGroup + AND JobType = OLD.JobType + AND ApplicationStatus = OLD.ApplicationStatus + AND MinorStatus = OLD.MinorStatus; + + -- Remove zero rows + DELETE FROM JobsHistorySummary + WHERE JobCount = 0 + AND Status = OLD.Status + AND Site = OLD.Site + AND Owner = OLD.Owner + AND OwnerGroup = OLD.OwnerGroup + AND VO = OLD.VO + AND JobGroup = OLD.JobGroup + AND JobType = OLD.JobType + AND ApplicationStatus = OLD.ApplicationStatus + AND MinorStatus = OLD.MinorStatus;""") + +trg_Jobs_update = Trigger( + name="trg_Jobs_update_status", + when="AFTER", + action="UPDATE", + table="Jobs", + time="FOR EACH ROW", + body="""\ + IF OLD.Status != NEW.Status THEN + + -- Decrease count from old status + UPDATE JobsHistorySummary + SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter + WHERE Status = OLD.Status + AND Site = OLD.Site + AND Owner = OLD.Owner + AND OwnerGroup = OLD.OwnerGroup + AND VO = OLD.VO + AND JobGroup = OLD.JobGroup + AND JobType = OLD.JobType + AND ApplicationStatus = OLD.ApplicationStatus + AND MinorStatus = OLD.MinorStatus; + + -- Delete row if count drops to zero + DELETE FROM JobsHistorySummary WHERE JobCount = 0; + + -- Increase count for new status + INSERT INTO JobsHistorySummary ( + Status, Site, Owner, OwnerGroup, JobGroup, VO, + JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum + ) + VALUES ( + NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup, + NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus, + 1, NEW.RescheduleCounter + ) + ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, + RescheduleSum = RescheduleSum + NEW.RescheduleCounter; + + END IF;""") + +trg_Jobs_insert.register_trigger(JobDBBase.metadata) +trg_Jobs_delete.register_trigger(JobDBBase.metadata) +trg_Jobs_update.register_trigger(JobDBBase.metadata) \ No newline at end of file diff --git a/diracx-db/src/diracx/db/sql/utils/__init__.py b/diracx-db/src/diracx/db/sql/utils/__init__.py index 69b78b4bf..946b70042 100644 --- a/diracx-db/src/diracx/db/sql/utils/__init__.py +++ b/diracx-db/src/diracx/db/sql/utils/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from .alembic import Trigger from .base import ( BaseSQLDB, SQLDBUnavailableError, @@ -22,4 +23,5 @@ "substract_date", "hash", "SQLDBUnavailableError", + "Trigger" ) diff --git a/diracx-db/src/diracx/db/sql/utils/alembic.py b/diracx-db/src/diracx/db/sql/utils/alembic.py new file mode 100644 index 000000000..f1b5fd7ce --- /dev/null +++ b/diracx-db/src/diracx/db/sql/utils/alembic.py @@ -0,0 +1,186 @@ +"""Alembic Utilities.""" +from __future__ import annotations + +from alembic.autogenerate import comparators, renderers +from alembic.operations import MigrateOperation, Operations +from sqlalchemy import text + +# Typing purposes +from sqlalchemy.sql.schema import MetaData + + +class Trigger: + """Creates trigger definitions in a format accepted by Alembic.""" + + def __init__(self, name: str, when: str, action: str, table: str, time: str, body: str): + self.name : str = name + self.when : str = when + self.action : str = action + self.table : str = table + self.time : str = time + self.body : str = body + + def create(self): + return f"""\ + CREATE TRIGGER {self.name} + {self.when} {self.action} + ON {self.table} + {self.time} + BEGIN + {self.body} + END""" + + def drop(self): + return f"DROP TRIGGER {self.name}" + + def __eq__(self, other: Trigger): + return self.name == other.name + + def __repr__(self): + return "Trigger(name=%r, when=%r, action=%r, table=%r, time=%r, body=%r)" \ + % (self.name, self.when, self.action, self.table, self.time, self.body) + + def register_trigger(self, metadata: MetaData): + """Add the trigger to database metadata.""" + metadata.info.setdefault("triggers", list()).append(self) + +# Custom Alemic Operations +########################## +@Operations.register_operation("create_trigger") +class CreateTriggerOp(MigrateOperation): + """Defines the operations to create triggers + Executed by calling op.create_trigger. + """ + + def __init__(self, trigger: Trigger): + self.trigger = trigger + + @classmethod + def create_trigger(cls, operations, name: str, **kw): + """Issue a "CREATE TRIGGER" instruction.""" + op = CreateTriggerOp(name, **kw) + return operations.invoke(op) + + def reverse(self): + # only needed to support autogenerate + return DropTriggerOp(self.trigger) + +@Operations.register_operation("drop_trigger") +class DropTriggerOp(MigrateOperation): + """Defines the operations to drop triggers + Executed by calling op.drop_trigger. + """ + + def __init__(self, trigger: Trigger): + self.trigger = trigger + + @classmethod + def drop_trigger(cls, operations, name: str, **kw): + """Issue a "DROP TRIGGER" instruction.""" + op = DropTriggerOp(name) + return operations.invoke(op) + + def reverse(self): + # only needed to support autogenerate + return CreateTriggerOp(self.trigger) + +@Operations.implementation_for(CreateTriggerOp) +def create_trigger(operations, operation): + """Receives a CreteTriggerOp operation and executes its sql text for its creation.""" + sql_text = operation.trigger.create() + operations.execute(sql_text) + +@Operations.implementation_for(DropTriggerOp) +def drop_trigger(operations, operation): + """Receives a DropTriggerOp operation and executes its sql text for its removal.""" + sql_text = operation.trigger.drop() + operations.execute(sql_text) + +# +# This function tells Alembic how to compare the state of the sqlalchemy matadata +# to the one found in the currently deployed database +# +# Due to triggers being stored in the database metadata, and not inside the table, the comparator +# executes at schema level. +@comparators.dispatch_for("schema") +def compare_triggers(autogen_context, operations, schemas): + """Compares the current state of the Metadata with the previous one found in the DB.""" + all_conn_triggers = list() + + # Get the collection of Triggers objects stored inside the metadata context + metadata_triggers = autogen_context.metadata.info.setdefault("triggers", list()) + + for schema in schemas: + # !!! This SQL Statement is MySQL specific !!!! + statement = text( + f"""SELECT TRIGGER_NAME, EVENT_MANIPULATION, ACTION_ORIENTATION, ACTION_TIMING, ACTION_STATEMENT \ + FROM information_schema.triggers \ + WHERE information_schema.triggers.trigger_schema \ + LIKE '{schema}';""") + + for row in autogen_context.connection.execute(statement): + trigger = Trigger( + name=row["TRIGGER_NAME"], + when=row['ACTION_TIMING'], + action=row['EVENT_MANIPULATION'], + table=schema, + time="FOR EACH ROW", + body=row['ACTION_STATEMENT'] + ) + + all_conn_triggers.append(trigger) + + # For new triggers found in the metadata + for trigger in metadata_triggers: + # The trigger cannot be already in the db + if trigger in all_conn_triggers: + continue + + # The trigger is new, so produce a CreateTriggerOp directive + operations.ops.append( + CreateTriggerOp(trigger) + ) + + # For triggers that are in the database + for trigger in all_conn_triggers: + # The trigger cannot be in the metadata + if trigger in metadata_triggers: + continue + + # The trigger got removed, so produce a DropTriggerOp directives + operations.ops.append( + DropTriggerOp(trigger) + ) + +# +# The renderer functions let alembic produce text that will be created inside the +# upgrade or downgrade action functions. +# +# This renderers also save some information inside an special dictionary called mutable_structure +# which let's us produce code inside the ".mako" template file +@renderers.dispatch_for(CreateTriggerOp) +def render_create_sequence(autogen_context, op: CreateTriggerOp): + """Almebic code renderer for CreateTrigger operations.""" + trigger = op.trigger + + ctx = autogen_context.opts['template_args']['mutable_structure']['triggers'] + + if trigger not in ctx: + ctx.append(trigger) + + # This part ends up being inside alembic's updagrade() or downgrade() functions + return f"op.create_trigger({trigger.name})" + + +@renderers.dispatch_for(DropTriggerOp) +def render_drop_sequence(autogen_context, op: DropTriggerOp): + """Almebic code renderer for DropTrigger operations.""" + trigger = op.trigger + + ctx = autogen_context.opts['template_args']['mutable_structure']['triggers'] + + if trigger not in ctx: + ctx.append(trigger) + + # This part ends up being inside alembic's updagrade() or downgrade() functions + return f"op.drop_trigger({trigger.name})" \ No newline at end of file From de2ec3c610e7f9b2e131ebdbe3eb201555ca2040 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 14:10:42 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../src/diracx/db/sql/job/migrations/README | 2 +- .../src/diracx/db/sql/job/migrations/env.py | 16 +- .../migrations/versions/79fce7dece6a_init.py | 264 ++++++++++-------- .../migrations/versions/b97381ed616f_new.py | 39 ++- diracx-db/src/diracx/db/sql/job/schema.py | 32 ++- diracx-db/src/diracx/db/sql/utils/__init__.py | 2 +- diracx-db/src/diracx/db/sql/utils/alembic.py | 80 +++--- 7 files changed, 253 insertions(+), 182 deletions(-) diff --git a/diracx-db/src/diracx/db/sql/job/migrations/README b/diracx-db/src/diracx/db/sql/job/migrations/README index e0d0858f2..a23d4fb51 100644 --- a/diracx-db/src/diracx/db/sql/job/migrations/README +++ b/diracx-db/src/diracx/db/sql/job/migrations/README @@ -1 +1 @@ -Generic single-database configuration with an async dbapi. \ No newline at end of file +Generic single-database configuration with an async dbapi. diff --git a/diracx-db/src/diracx/db/sql/job/migrations/env.py b/diracx-db/src/diracx/db/sql/job/migrations/env.py index 37f70f776..e48bc5b7e 100644 --- a/diracx-db/src/diracx/db/sql/job/migrations/env.py +++ b/diracx-db/src/diracx/db/sql/job/migrations/env.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio from logging.config import fileConfig @@ -7,14 +9,13 @@ from sqlalchemy.ext.asyncio import async_engine_from_config from diracx.db.sql.job.schema import JobDBBase as Base -from diracx.db.sql.utils.base import BaseSQLDB # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config -url = "mysql+aiomysql://user:password@localhost:3306/JobDB" -#url = BaseSQLDB.available_urls()["JobDB"] #! This Env Variable refers to a non root user, which doesn't have enough permissions +url = "mysql+aiomysql://user:password@localhost:3306/JobDB" +# url = BaseSQLDB.available_urls()["JobDB"] #! This Env Variable refers to a non root user, which doesn't have enough permissions config.set_main_option("sqlalchemy.url", url) # Interpret the config file for Python logging. @@ -33,6 +34,7 @@ # my_important_option = config.get_main_option("my_important_option") # ... etc. + def run_migrations_offline() -> None: """Run migrations in 'offline' mode. @@ -59,10 +61,10 @@ def run_migrations_offline() -> None: def do_run_migrations(connection: Connection) -> None: context.configure( - connection=connection, - target_metadata=target_metadata, - # The mutable_structure lets store some extra information for future rendering - template_args={"mutable_structure": {"triggers": []}} + connection=connection, + target_metadata=target_metadata, + # The mutable_structure lets store some extra information for future rendering + template_args={"mutable_structure": {"triggers": []}}, ) with context.begin_transaction(): diff --git a/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py b/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py index 57183d966..af3dcf1d7 100644 --- a/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py +++ b/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py @@ -1,18 +1,22 @@ """init Revision ID: 79fce7dece6a -Revises: +Revises: Create Date: 2025-06-20 16:31:53.704617 """ + +from __future__ import annotations + from typing import Sequence, Union import sqlalchemy as sa from alembic import op + import diracx # revision identifiers, used by Alembic. -revision: str = '79fce7dece6a' +revision: str = "79fce7dece6a" down_revision: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -21,110 +25,136 @@ def upgrade() -> None: """Upgrade schema.""" # ### commands auto generated by Alembic - please adjust! ### - op.create_table('JobJDLs', - sa.Column('JobID', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('JDL', sa.Text(), nullable=False), - sa.Column('JobRequirements', sa.Text(), nullable=False), - sa.Column('OriginalJDL', sa.Text(), nullable=False), - sa.PrimaryKeyConstraint('JobID') + op.create_table( + "JobJDLs", + sa.Column("JobID", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("JDL", sa.Text(), nullable=False), + sa.Column("JobRequirements", sa.Text(), nullable=False), + sa.Column("OriginalJDL", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("JobID"), ) - op.create_table('JobsHistorySummary', - sa.Column('ID', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('Status', sa.String(length=32), nullable=False), - sa.Column('Site', sa.String(length=100), nullable=False), - sa.Column('Owner', sa.String(length=64), nullable=False), - sa.Column('OwnerGroup', sa.String(length=128), nullable=False), - sa.Column('VO', sa.String(length=64), nullable=False), - sa.Column('JobGroup', sa.String(length=32), nullable=False), - sa.Column('JobType', sa.String(length=32), nullable=False), - sa.Column('ApplicationStatus', sa.String(length=255), nullable=False), - sa.Column('MinorStatus', sa.String(length=128), nullable=False), - sa.Column('JobCount', sa.Integer(), nullable=False), - sa.Column('RescheduleSum', sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint('ID'), - sa.UniqueConstraint('Status', 'Site', 'Owner', 'OwnerGroup', 'VO', 'JobGroup', 'JobType', 'ApplicationStatus', 'MinorStatus', name='uq_summary') + op.create_table( + "JobsHistorySummary", + sa.Column("ID", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.Column("Site", sa.String(length=100), nullable=False), + sa.Column("Owner", sa.String(length=64), nullable=False), + sa.Column("OwnerGroup", sa.String(length=128), nullable=False), + sa.Column("VO", sa.String(length=64), nullable=False), + sa.Column("JobGroup", sa.String(length=32), nullable=False), + sa.Column("JobType", sa.String(length=32), nullable=False), + sa.Column("ApplicationStatus", sa.String(length=255), nullable=False), + sa.Column("MinorStatus", sa.String(length=128), nullable=False), + sa.Column("JobCount", sa.Integer(), nullable=False), + sa.Column("RescheduleSum", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("ID"), + sa.UniqueConstraint( + "Status", + "Site", + "Owner", + "OwnerGroup", + "VO", + "JobGroup", + "JobType", + "ApplicationStatus", + "MinorStatus", + name="uq_summary", + ), ) - op.create_table('Jobs', - sa.Column('JobID', sa.Integer(), nullable=False), - sa.Column('JobType', sa.String(length=32), nullable=False), - sa.Column('JobGroup', sa.String(length=32), nullable=False), - sa.Column('Site', sa.String(length=100), nullable=False), - sa.Column('JobName', sa.String(length=128), nullable=False), - sa.Column('Owner', sa.String(length=64), nullable=False), - sa.Column('OwnerGroup', sa.String(length=128), nullable=False), - sa.Column('VO', sa.String(length=64), nullable=False), - sa.Column('SubmissionTime', sa.DateTime(), nullable=True), - sa.Column('RescheduleTime', sa.DateTime(), nullable=True), - sa.Column('LastUpdateTime', sa.DateTime(), nullable=True), - sa.Column('StartExecTime', sa.DateTime(), nullable=True), - sa.Column('HeartBeatTime', sa.DateTime(), nullable=True), - sa.Column('EndExecTime', sa.DateTime(), nullable=True), - sa.Column('Status', sa.String(length=32), nullable=False), - sa.Column('MinorStatus', sa.String(length=128), nullable=False), - sa.Column('ApplicationStatus', sa.String(length=255), nullable=False), - sa.Column('UserPriority', sa.Integer(), nullable=False), - sa.Column('RescheduleCounter', sa.Integer(), nullable=False), - sa.Column('VerifiedFlag', diracx.db.sql.utils.types.EnumBackedBool(), nullable=False), - sa.Column('AccountedFlag', diracx.db.sql.job.schema.AccountedFlagEnum(), nullable=False), - sa.ForeignKeyConstraint(['JobID'], ['JobJDLs.JobID'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('JobID') + op.create_table( + "Jobs", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("JobType", sa.String(length=32), nullable=False), + sa.Column("JobGroup", sa.String(length=32), nullable=False), + sa.Column("Site", sa.String(length=100), nullable=False), + sa.Column("JobName", sa.String(length=128), nullable=False), + sa.Column("Owner", sa.String(length=64), nullable=False), + sa.Column("OwnerGroup", sa.String(length=128), nullable=False), + sa.Column("VO", sa.String(length=64), nullable=False), + sa.Column("SubmissionTime", sa.DateTime(), nullable=True), + sa.Column("RescheduleTime", sa.DateTime(), nullable=True), + sa.Column("LastUpdateTime", sa.DateTime(), nullable=True), + sa.Column("StartExecTime", sa.DateTime(), nullable=True), + sa.Column("HeartBeatTime", sa.DateTime(), nullable=True), + sa.Column("EndExecTime", sa.DateTime(), nullable=True), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.Column("MinorStatus", sa.String(length=128), nullable=False), + sa.Column("ApplicationStatus", sa.String(length=255), nullable=False), + sa.Column("UserPriority", sa.Integer(), nullable=False), + sa.Column("RescheduleCounter", sa.Integer(), nullable=False), + sa.Column( + "VerifiedFlag", diracx.db.sql.utils.types.EnumBackedBool(), nullable=False + ), + sa.Column( + "AccountedFlag", + diracx.db.sql.job.schema.AccountedFlagEnum(), + nullable=False, + ), + sa.ForeignKeyConstraint(["JobID"], ["JobJDLs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID"), ) - op.create_index('ApplicationStatus', 'Jobs', ['ApplicationStatus'], unique=False) - op.create_index('JobGroup', 'Jobs', ['JobGroup'], unique=False) - op.create_index('JobType', 'Jobs', ['JobType'], unique=False) - op.create_index('LastUpdateTime', 'Jobs', ['LastUpdateTime'], unique=False) - op.create_index('MinorStatus', 'Jobs', ['MinorStatus'], unique=False) - op.create_index('Owner', 'Jobs', ['Owner'], unique=False) - op.create_index('OwnerGroup', 'Jobs', ['OwnerGroup'], unique=False) - op.create_index('Site', 'Jobs', ['Site'], unique=False) - op.create_index('Status', 'Jobs', ['Status'], unique=False) - op.create_index('StatusSite', 'Jobs', ['Status', 'Site'], unique=False) - op.create_table('AtticJobParameters', - sa.Column('JobID', sa.Integer(), nullable=False), - sa.Column('Name', sa.String(length=100), nullable=False), - sa.Column('Value', sa.Text(), nullable=False), - sa.Column('RescheduleCycle', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('JobID', 'Name') + op.create_index("ApplicationStatus", "Jobs", ["ApplicationStatus"], unique=False) + op.create_index("JobGroup", "Jobs", ["JobGroup"], unique=False) + op.create_index("JobType", "Jobs", ["JobType"], unique=False) + op.create_index("LastUpdateTime", "Jobs", ["LastUpdateTime"], unique=False) + op.create_index("MinorStatus", "Jobs", ["MinorStatus"], unique=False) + op.create_index("Owner", "Jobs", ["Owner"], unique=False) + op.create_index("OwnerGroup", "Jobs", ["OwnerGroup"], unique=False) + op.create_index("Site", "Jobs", ["Site"], unique=False) + op.create_index("Status", "Jobs", ["Status"], unique=False) + op.create_index("StatusSite", "Jobs", ["Status", "Site"], unique=False) + op.create_table( + "AtticJobParameters", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.Column("RescheduleCycle", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name"), ) - op.create_table('HeartBeatLoggingInfo', - sa.Column('JobID', sa.Integer(), nullable=False), - sa.Column('Name', sa.String(length=100), nullable=False), - sa.Column('Value', sa.Text(), nullable=False), - sa.Column('HeartBeatTime', sa.DateTime(), nullable=False), - sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('JobID', 'Name', 'HeartBeatTime') + op.create_table( + "HeartBeatLoggingInfo", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.Column("HeartBeatTime", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name", "HeartBeatTime"), ) - op.create_table('InputData', - sa.Column('JobID', sa.Integer(), nullable=False), - sa.Column('LFN', sa.String(length=255), nullable=False), - sa.Column('Status', sa.String(length=32), nullable=False), - sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('JobID', 'LFN') + op.create_table( + "InputData", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("LFN", sa.String(length=255), nullable=False), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "LFN"), ) - op.create_table('JobCommands', - sa.Column('JobID', sa.Integer(), nullable=False), - sa.Column('Command', sa.String(length=100), nullable=False), - sa.Column('Arguments', sa.String(length=100), nullable=False), - sa.Column('Status', sa.String(length=32), nullable=False), - sa.Column('ReceptionTime', sa.DateTime(), nullable=False), - sa.Column('ExecutionTime', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('JobID', 'ReceptionTime') + op.create_table( + "JobCommands", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Command", sa.String(length=100), nullable=False), + sa.Column("Arguments", sa.String(length=100), nullable=False), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.Column("ReceptionTime", sa.DateTime(), nullable=False), + sa.Column("ExecutionTime", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "ReceptionTime"), ) - op.create_table('JobParameters', - sa.Column('JobID', sa.Integer(), nullable=False), - sa.Column('Name', sa.String(length=100), nullable=False), - sa.Column('Value', sa.Text(), nullable=False), - sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('JobID', 'Name') + op.create_table( + "JobParameters", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name"), ) - op.create_table('OptimizerParameters', - sa.Column('JobID', sa.Integer(), nullable=False), - sa.Column('Name', sa.String(length=100), nullable=False), - sa.Column('Value', sa.Text(), nullable=False), - sa.ForeignKeyConstraint(['JobID'], ['Jobs.JobID'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('JobID', 'Name') + op.create_table( + "OptimizerParameters", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name"), ) # ### end Alembic commands ### @@ -132,23 +162,23 @@ def upgrade() -> None: def downgrade() -> None: """Downgrade schema.""" # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('OptimizerParameters') - op.drop_table('JobParameters') - op.drop_table('JobCommands') - op.drop_table('InputData') - op.drop_table('HeartBeatLoggingInfo') - op.drop_table('AtticJobParameters') - op.drop_index('StatusSite', table_name='Jobs') - op.drop_index('Status', table_name='Jobs') - op.drop_index('Site', table_name='Jobs') - op.drop_index('OwnerGroup', table_name='Jobs') - op.drop_index('Owner', table_name='Jobs') - op.drop_index('MinorStatus', table_name='Jobs') - op.drop_index('LastUpdateTime', table_name='Jobs') - op.drop_index('JobType', table_name='Jobs') - op.drop_index('JobGroup', table_name='Jobs') - op.drop_index('ApplicationStatus', table_name='Jobs') - op.drop_table('Jobs') - op.drop_table('JobsHistorySummary') - op.drop_table('JobJDLs') + op.drop_table("OptimizerParameters") + op.drop_table("JobParameters") + op.drop_table("JobCommands") + op.drop_table("InputData") + op.drop_table("HeartBeatLoggingInfo") + op.drop_table("AtticJobParameters") + op.drop_index("StatusSite", table_name="Jobs") + op.drop_index("Status", table_name="Jobs") + op.drop_index("Site", table_name="Jobs") + op.drop_index("OwnerGroup", table_name="Jobs") + op.drop_index("Owner", table_name="Jobs") + op.drop_index("MinorStatus", table_name="Jobs") + op.drop_index("LastUpdateTime", table_name="Jobs") + op.drop_index("JobType", table_name="Jobs") + op.drop_index("JobGroup", table_name="Jobs") + op.drop_index("ApplicationStatus", table_name="Jobs") + op.drop_table("Jobs") + op.drop_table("JobsHistorySummary") + op.drop_table("JobJDLs") # ### end Alembic commands ### diff --git a/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py b/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py index b9ac408fa..0560d812d 100644 --- a/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py +++ b/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py @@ -5,23 +5,46 @@ Create Date: 2025-07-01 12:41:17.490790 """ + +from __future__ import annotations + from typing import Sequence, Union from alembic import op -import sqlalchemy as sa -import diracx -from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. -revision: str = 'b97381ed616f' -down_revision: Union[str, Sequence[str], None] = '79fce7dece6a' +revision: str = "b97381ed616f" +down_revision: Union[str, Sequence[str], None] = "79fce7dece6a" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None from diracx.db.sql.utils.alembic import Trigger -trg_Jobs_insert = Trigger(name='trg_Jobs_insert', when='AFTER', action='INSERT', table='Jobs', time='FOR EACH ROW', body=' INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType,\n ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO,\n NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus,\n NEW.MinorStatus, 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;') -trg_Jobs_delete = Trigger(name='trg_Jobs_delete', when='AFTER', action='DELETE', table='Jobs', time='FOR EACH ROW', body=' UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Remove zero rows\n DELETE FROM JobsHistorySummary\n WHERE JobCount = 0\n AND Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;') -trg_Jobs_update_status = Trigger(name='trg_Jobs_update_status', when='AFTER', action='UPDATE', table='Jobs', time='FOR EACH ROW', body=' IF OLD.Status != NEW.Status THEN\n\n -- Decrease count from old status\n UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Delete row if count drops to zero\n DELETE FROM JobsHistorySummary WHERE JobCount = 0;\n\n -- Increase count for new status\n INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, JobGroup, VO,\n JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup,\n NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus,\n 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;\n\n END IF;') + +trg_Jobs_insert = Trigger( + name="trg_Jobs_insert", + when="AFTER", + action="INSERT", + table="Jobs", + time="FOR EACH ROW", + body=" INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType,\n ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO,\n NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus,\n NEW.MinorStatus, 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;", +) +trg_Jobs_delete = Trigger( + name="trg_Jobs_delete", + when="AFTER", + action="DELETE", + table="Jobs", + time="FOR EACH ROW", + body=" UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Remove zero rows\n DELETE FROM JobsHistorySummary\n WHERE JobCount = 0\n AND Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;", +) +trg_Jobs_update_status = Trigger( + name="trg_Jobs_update_status", + when="AFTER", + action="UPDATE", + table="Jobs", + time="FOR EACH ROW", + body=" IF OLD.Status != NEW.Status THEN\n\n -- Decrease count from old status\n UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Delete row if count drops to zero\n DELETE FROM JobsHistorySummary WHERE JobCount = 0;\n\n -- Increase count for new status\n INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, JobGroup, VO,\n JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup,\n NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus,\n 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;\n\n END IF;", +) + def upgrade() -> None: """Upgrade schema.""" diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index 6784eaf17..b6e67d436 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -189,6 +189,7 @@ class JobsHistorySummary(JobDBBase): ), ) + ########################################################### # Triggers defined for SqlAlchemy ########################################################### @@ -293,16 +294,16 @@ class JobsHistorySummary(JobDBBase): ########################################################### # Triggers defined for Alembic ########################################################### -# If we use Alembic, we need a class prepared for it to +# If we use Alembic, we need a class prepared for it to # understand, using ReplaceableObjects. # This creates the triggers at SQL level, inside the db. ########################################################### trg_Jobs_insert = Trigger( name="trg_Jobs_insert", - when="AFTER", - action="INSERT", - table="Jobs", + when="AFTER", + action="INSERT", + table="Jobs", time="FOR EACH ROW", body="""\ INSERT INTO JobsHistorySummary ( @@ -315,13 +316,14 @@ class JobsHistorySummary(JobDBBase): NEW.MinorStatus, 1, NEW.RescheduleCounter ) ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, - RescheduleSum = RescheduleSum + NEW.RescheduleCounter;""") + RescheduleSum = RescheduleSum + NEW.RescheduleCounter;""", +) trg_Jobs_delete = Trigger( name="trg_Jobs_delete", - when="AFTER", - action="DELETE", - table="Jobs", + when="AFTER", + action="DELETE", + table="Jobs", time="FOR EACH ROW", body="""\ UPDATE JobsHistorySummary @@ -347,13 +349,14 @@ class JobsHistorySummary(JobDBBase): AND JobGroup = OLD.JobGroup AND JobType = OLD.JobType AND ApplicationStatus = OLD.ApplicationStatus - AND MinorStatus = OLD.MinorStatus;""") + AND MinorStatus = OLD.MinorStatus;""", +) trg_Jobs_update = Trigger( name="trg_Jobs_update_status", - when="AFTER", - action="UPDATE", - table="Jobs", + when="AFTER", + action="UPDATE", + table="Jobs", time="FOR EACH ROW", body="""\ IF OLD.Status != NEW.Status THEN @@ -387,8 +390,9 @@ class JobsHistorySummary(JobDBBase): ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, RescheduleSum = RescheduleSum + NEW.RescheduleCounter; - END IF;""") + END IF;""", +) trg_Jobs_insert.register_trigger(JobDBBase.metadata) trg_Jobs_delete.register_trigger(JobDBBase.metadata) -trg_Jobs_update.register_trigger(JobDBBase.metadata) \ No newline at end of file +trg_Jobs_update.register_trigger(JobDBBase.metadata) diff --git a/diracx-db/src/diracx/db/sql/utils/__init__.py b/diracx-db/src/diracx/db/sql/utils/__init__.py index 946b70042..757913dfc 100644 --- a/diracx-db/src/diracx/db/sql/utils/__init__.py +++ b/diracx-db/src/diracx/db/sql/utils/__init__.py @@ -23,5 +23,5 @@ "substract_date", "hash", "SQLDBUnavailableError", - "Trigger" + "Trigger", ) diff --git a/diracx-db/src/diracx/db/sql/utils/alembic.py b/diracx-db/src/diracx/db/sql/utils/alembic.py index f1b5fd7ce..c53832508 100644 --- a/diracx-db/src/diracx/db/sql/utils/alembic.py +++ b/diracx-db/src/diracx/db/sql/utils/alembic.py @@ -1,4 +1,5 @@ """Alembic Utilities.""" + from __future__ import annotations from alembic.autogenerate import comparators, renderers @@ -12,13 +13,15 @@ class Trigger: """Creates trigger definitions in a format accepted by Alembic.""" - def __init__(self, name: str, when: str, action: str, table: str, time: str, body: str): - self.name : str = name - self.when : str = when - self.action : str = action - self.table : str = table - self.time : str = time - self.body : str = body + def __init__( + self, name: str, when: str, action: str, table: str, time: str, body: str + ): + self.name: str = name + self.when: str = when + self.action: str = action + self.table: str = table + self.time: str = time + self.body: str = body def create(self): return f"""\ @@ -29,21 +32,28 @@ def create(self): BEGIN {self.body} END""" - + def drop(self): return f"DROP TRIGGER {self.name}" - + def __eq__(self, other: Trigger): return self.name == other.name - + def __repr__(self): - return "Trigger(name=%r, when=%r, action=%r, table=%r, time=%r, body=%r)" \ - % (self.name, self.when, self.action, self.table, self.time, self.body) + return "Trigger(name=%r, when=%r, action=%r, table=%r, time=%r, body=%r)" % ( + self.name, + self.when, + self.action, + self.table, + self.time, + self.body, + ) def register_trigger(self, metadata: MetaData): """Add the trigger to database metadata.""" metadata.info.setdefault("triggers", list()).append(self) + # Custom Alemic Operations ########################## @Operations.register_operation("create_trigger") @@ -65,6 +75,7 @@ def reverse(self): # only needed to support autogenerate return DropTriggerOp(self.trigger) + @Operations.register_operation("drop_trigger") class DropTriggerOp(MigrateOperation): """Defines the operations to drop triggers @@ -84,20 +95,23 @@ def reverse(self): # only needed to support autogenerate return CreateTriggerOp(self.trigger) + @Operations.implementation_for(CreateTriggerOp) def create_trigger(operations, operation): """Receives a CreteTriggerOp operation and executes its sql text for its creation.""" sql_text = operation.trigger.create() operations.execute(sql_text) + @Operations.implementation_for(DropTriggerOp) def drop_trigger(operations, operation): """Receives a DropTriggerOp operation and executes its sql text for its removal.""" sql_text = operation.trigger.drop() operations.execute(sql_text) + # -# This function tells Alembic how to compare the state of the sqlalchemy matadata +# This function tells Alembic how to compare the state of the sqlalchemy metadata # to the one found in the currently deployed database # # Due to triggers being stored in the database metadata, and not inside the table, the comparator @@ -116,20 +130,21 @@ def compare_triggers(autogen_context, operations, schemas): f"""SELECT TRIGGER_NAME, EVENT_MANIPULATION, ACTION_ORIENTATION, ACTION_TIMING, ACTION_STATEMENT \ FROM information_schema.triggers \ WHERE information_schema.triggers.trigger_schema \ - LIKE '{schema}';""") + LIKE '{schema}';""" + ) - for row in autogen_context.connection.execute(statement): + for row in autogen_context.connection.execute(statement): trigger = Trigger( name=row["TRIGGER_NAME"], - when=row['ACTION_TIMING'], - action=row['EVENT_MANIPULATION'], + when=row["ACTION_TIMING"], + action=row["EVENT_MANIPULATION"], table=schema, time="FOR EACH ROW", - body=row['ACTION_STATEMENT'] + body=row["ACTION_STATEMENT"], ) all_conn_triggers.append(trigger) - + # For new triggers found in the metadata for trigger in metadata_triggers: # The trigger cannot be already in the db @@ -137,9 +152,7 @@ def compare_triggers(autogen_context, operations, schemas): continue # The trigger is new, so produce a CreateTriggerOp directive - operations.ops.append( - CreateTriggerOp(trigger) - ) + operations.ops.append(CreateTriggerOp(trigger)) # For triggers that are in the database for trigger in all_conn_triggers: @@ -148,26 +161,25 @@ def compare_triggers(autogen_context, operations, schemas): continue # The trigger got removed, so produce a DropTriggerOp directives - operations.ops.append( - DropTriggerOp(trigger) - ) + operations.ops.append(DropTriggerOp(trigger)) + # -# The renderer functions let alembic produce text that will be created inside the +# The renderer functions let alembic produce text that will be created inside the # upgrade or downgrade action functions. -# +# # This renderers also save some information inside an special dictionary called mutable_structure -# which let's us produce code inside the ".mako" template file +# which let's us produce code inside the ".mako" template file @renderers.dispatch_for(CreateTriggerOp) def render_create_sequence(autogen_context, op: CreateTriggerOp): """Almebic code renderer for CreateTrigger operations.""" trigger = op.trigger - - ctx = autogen_context.opts['template_args']['mutable_structure']['triggers'] - + + ctx = autogen_context.opts["template_args"]["mutable_structure"]["triggers"] + if trigger not in ctx: ctx.append(trigger) - + # This part ends up being inside alembic's updagrade() or downgrade() functions return f"op.create_trigger({trigger.name})" @@ -177,10 +189,10 @@ def render_drop_sequence(autogen_context, op: DropTriggerOp): """Almebic code renderer for DropTrigger operations.""" trigger = op.trigger - ctx = autogen_context.opts['template_args']['mutable_structure']['triggers'] + ctx = autogen_context.opts["template_args"]["mutable_structure"]["triggers"] if trigger not in ctx: ctx.append(trigger) # This part ends up being inside alembic's updagrade() or downgrade() functions - return f"op.drop_trigger({trigger.name})" \ No newline at end of file + return f"op.drop_trigger({trigger.name})"