diff --git a/diracx-db/pyproject.toml b/diracx-db/pyproject.toml index 8a5e87d8a..e7c673c3c 100644 --- a/diracx-db/pyproject.toml +++ b/diracx-db/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "pydantic >=2.10", "sqlalchemy[aiomysql,aiosqlite] >= 2", "uuid-utils", + "alembic" ] dynamic = ["version"] diff --git a/diracx-db/src/diracx/db/sql/job/alembic.ini b/diracx-db/src/diracx/db/sql/job/alembic.ini new file mode 100644 index 000000000..cf1177faf --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/alembic.ini @@ -0,0 +1,141 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/diracx-db/src/diracx/db/sql/job/migrations/README b/diracx-db/src/diracx/db/sql/job/migrations/README new file mode 100644 index 000000000..a23d4fb51 --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. diff --git a/diracx-db/src/diracx/db/sql/job/migrations/env.py b/diracx-db/src/diracx/db/sql/job/migrations/env.py new file mode 100644 index 000000000..e48bc5b7e --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/env.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from diracx.db.sql.job.schema import JobDBBase as Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +url = "mysql+aiomysql://user:password@localhost:3306/JobDB" +# url = BaseSQLDB.available_urls()["JobDB"] #! This Env Variable refers to a non root user, which doesn't have enough permissions +config.set_main_option("sqlalchemy.url", url) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + # The mutable_structure lets store some extra information for future rendering + template_args={"mutable_structure": {"triggers": []}}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """In this scenario we need to create an Engine + and associate a connection with the context. + """ + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/diracx-db/src/diracx/db/sql/job/migrations/script.py.mako b/diracx-db/src/diracx/db/sql/job/migrations/script.py.mako new file mode 100644 index 000000000..08dc41dcc --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/script.py.mako @@ -0,0 +1,36 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import diracx +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + +% if mutable_structure['triggers']: +# Trigger definitions +from diracx.db.sql.utils.alembic import Trigger +% for trigger in mutable_structure['triggers']: +${trigger.name} = ${repr(trigger)} +% endfor +% endif + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py b/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py new file mode 100644 index 000000000..af3dcf1d7 --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/versions/79fce7dece6a_init.py @@ -0,0 +1,184 @@ +"""init + +Revision ID: 79fce7dece6a +Revises: +Create Date: 2025-06-20 16:31:53.704617 + +""" + +from __future__ import annotations + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +import diracx + +# revision identifiers, used by Alembic. +revision: str = "79fce7dece6a" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "JobJDLs", + sa.Column("JobID", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("JDL", sa.Text(), nullable=False), + sa.Column("JobRequirements", sa.Text(), nullable=False), + sa.Column("OriginalJDL", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("JobID"), + ) + op.create_table( + "JobsHistorySummary", + sa.Column("ID", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.Column("Site", sa.String(length=100), nullable=False), + sa.Column("Owner", sa.String(length=64), nullable=False), + sa.Column("OwnerGroup", sa.String(length=128), nullable=False), + sa.Column("VO", sa.String(length=64), nullable=False), + sa.Column("JobGroup", sa.String(length=32), nullable=False), + sa.Column("JobType", sa.String(length=32), nullable=False), + sa.Column("ApplicationStatus", sa.String(length=255), nullable=False), + sa.Column("MinorStatus", sa.String(length=128), nullable=False), + sa.Column("JobCount", sa.Integer(), nullable=False), + sa.Column("RescheduleSum", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("ID"), + sa.UniqueConstraint( + "Status", + "Site", + "Owner", + "OwnerGroup", + "VO", + "JobGroup", + "JobType", + "ApplicationStatus", + "MinorStatus", + name="uq_summary", + ), + ) + op.create_table( + "Jobs", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("JobType", sa.String(length=32), nullable=False), + sa.Column("JobGroup", sa.String(length=32), nullable=False), + sa.Column("Site", sa.String(length=100), nullable=False), + sa.Column("JobName", sa.String(length=128), nullable=False), + sa.Column("Owner", sa.String(length=64), nullable=False), + sa.Column("OwnerGroup", sa.String(length=128), nullable=False), + sa.Column("VO", sa.String(length=64), nullable=False), + sa.Column("SubmissionTime", sa.DateTime(), nullable=True), + sa.Column("RescheduleTime", sa.DateTime(), nullable=True), + sa.Column("LastUpdateTime", sa.DateTime(), nullable=True), + sa.Column("StartExecTime", sa.DateTime(), nullable=True), + sa.Column("HeartBeatTime", sa.DateTime(), nullable=True), + sa.Column("EndExecTime", sa.DateTime(), nullable=True), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.Column("MinorStatus", sa.String(length=128), nullable=False), + sa.Column("ApplicationStatus", sa.String(length=255), nullable=False), + sa.Column("UserPriority", sa.Integer(), nullable=False), + sa.Column("RescheduleCounter", sa.Integer(), nullable=False), + sa.Column( + "VerifiedFlag", diracx.db.sql.utils.types.EnumBackedBool(), nullable=False + ), + sa.Column( + "AccountedFlag", + diracx.db.sql.job.schema.AccountedFlagEnum(), + nullable=False, + ), + sa.ForeignKeyConstraint(["JobID"], ["JobJDLs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID"), + ) + op.create_index("ApplicationStatus", "Jobs", ["ApplicationStatus"], unique=False) + op.create_index("JobGroup", "Jobs", ["JobGroup"], unique=False) + op.create_index("JobType", "Jobs", ["JobType"], unique=False) + op.create_index("LastUpdateTime", "Jobs", ["LastUpdateTime"], unique=False) + op.create_index("MinorStatus", "Jobs", ["MinorStatus"], unique=False) + op.create_index("Owner", "Jobs", ["Owner"], unique=False) + op.create_index("OwnerGroup", "Jobs", ["OwnerGroup"], unique=False) + op.create_index("Site", "Jobs", ["Site"], unique=False) + op.create_index("Status", "Jobs", ["Status"], unique=False) + op.create_index("StatusSite", "Jobs", ["Status", "Site"], unique=False) + op.create_table( + "AtticJobParameters", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.Column("RescheduleCycle", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name"), + ) + op.create_table( + "HeartBeatLoggingInfo", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.Column("HeartBeatTime", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name", "HeartBeatTime"), + ) + op.create_table( + "InputData", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("LFN", sa.String(length=255), nullable=False), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "LFN"), + ) + op.create_table( + "JobCommands", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Command", sa.String(length=100), nullable=False), + sa.Column("Arguments", sa.String(length=100), nullable=False), + sa.Column("Status", sa.String(length=32), nullable=False), + sa.Column("ReceptionTime", sa.DateTime(), nullable=False), + sa.Column("ExecutionTime", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "ReceptionTime"), + ) + op.create_table( + "JobParameters", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name"), + ) + op.create_table( + "OptimizerParameters", + sa.Column("JobID", sa.Integer(), nullable=False), + sa.Column("Name", sa.String(length=100), nullable=False), + sa.Column("Value", sa.Text(), nullable=False), + sa.ForeignKeyConstraint(["JobID"], ["Jobs.JobID"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("JobID", "Name"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("OptimizerParameters") + op.drop_table("JobParameters") + op.drop_table("JobCommands") + op.drop_table("InputData") + op.drop_table("HeartBeatLoggingInfo") + op.drop_table("AtticJobParameters") + op.drop_index("StatusSite", table_name="Jobs") + op.drop_index("Status", table_name="Jobs") + op.drop_index("Site", table_name="Jobs") + op.drop_index("OwnerGroup", table_name="Jobs") + op.drop_index("Owner", table_name="Jobs") + op.drop_index("MinorStatus", table_name="Jobs") + op.drop_index("LastUpdateTime", table_name="Jobs") + op.drop_index("JobType", table_name="Jobs") + op.drop_index("JobGroup", table_name="Jobs") + op.drop_index("ApplicationStatus", table_name="Jobs") + op.drop_table("Jobs") + op.drop_table("JobsHistorySummary") + op.drop_table("JobJDLs") + # ### end Alembic commands ### diff --git a/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py b/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py new file mode 100644 index 000000000..0560d812d --- /dev/null +++ b/diracx-db/src/diracx/db/sql/job/migrations/versions/b97381ed616f_new.py @@ -0,0 +1,64 @@ +"""new + +Revision ID: b97381ed616f +Revises: 79fce7dece6a +Create Date: 2025-07-01 12:41:17.490790 + +""" + +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "b97381ed616f" +down_revision: Union[str, Sequence[str], None] = "79fce7dece6a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +from diracx.db.sql.utils.alembic import Trigger + +trg_Jobs_insert = Trigger( + name="trg_Jobs_insert", + when="AFTER", + action="INSERT", + table="Jobs", + time="FOR EACH ROW", + body=" INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType,\n ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO,\n NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus,\n NEW.MinorStatus, 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;", +) +trg_Jobs_delete = Trigger( + name="trg_Jobs_delete", + when="AFTER", + action="DELETE", + table="Jobs", + time="FOR EACH ROW", + body=" UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Remove zero rows\n DELETE FROM JobsHistorySummary\n WHERE JobCount = 0\n AND Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;", +) +trg_Jobs_update_status = Trigger( + name="trg_Jobs_update_status", + when="AFTER", + action="UPDATE", + table="Jobs", + time="FOR EACH ROW", + body=" IF OLD.Status != NEW.Status THEN\n\n -- Decrease count from old status\n UPDATE JobsHistorySummary\n SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter\n WHERE Status = OLD.Status\n AND Site = OLD.Site\n AND Owner = OLD.Owner\n AND OwnerGroup = OLD.OwnerGroup\n AND VO = OLD.VO\n AND JobGroup = OLD.JobGroup\n AND JobType = OLD.JobType\n AND ApplicationStatus = OLD.ApplicationStatus\n AND MinorStatus = OLD.MinorStatus;\n\n -- Delete row if count drops to zero\n DELETE FROM JobsHistorySummary WHERE JobCount = 0;\n\n -- Increase count for new status\n INSERT INTO JobsHistorySummary (\n Status, Site, Owner, OwnerGroup, JobGroup, VO,\n JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum\n )\n VALUES (\n NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup,\n NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus,\n 1, NEW.RescheduleCounter\n )\n ON DUPLICATE KEY UPDATE JobCount = JobCount + 1,\n RescheduleSum = RescheduleSum + NEW.RescheduleCounter;\n\n END IF;", +) + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_trigger(trg_Jobs_insert) + op.create_trigger(trg_Jobs_delete) + op.create_trigger(trg_Jobs_update_status) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_trigger(trg_Jobs_update_status) + op.drop_trigger(trg_Jobs_delete) + op.drop_trigger(trg_Jobs_insert) + # ### end Alembic commands ### diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index bb9f60bf1..b6e67d436 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -8,10 +8,11 @@ Integer, String, Text, + UniqueConstraint, ) from sqlalchemy.orm import declarative_base -from ..utils import Column, EnumBackedBool, NullColumn +from ..utils import Column, EnumBackedBool, NullColumn, Trigger JobDBBase = declarative_base() @@ -62,7 +63,7 @@ class Jobs(JobDBBase): job_name = Column("JobName", String(128), default="Unknown") owner = Column("Owner", String(64), default="Unknown") owner_group = Column("OwnerGroup", String(128), default="Unknown") - vo = Column("VO", String(32)) + vo = Column("VO", String(64)) submission_time = NullColumn("SubmissionTime", DateTime) reschedule_time = NullColumn("RescheduleTime", DateTime) last_update_time = NullColumn("LastUpdateTime", DateTime) @@ -153,6 +154,245 @@ class JobCommands(JobDBBase): ) command = Column("Command", String(100)) arguments = Column("Arguments", String(100)) - status = Column("Status", String(64), default="Received") + status = Column("Status", String(32), default="Received") reception_time = Column("ReceptionTime", DateTime, primary_key=True) execution_time = NullColumn("ExecutionTime", DateTime) + + +class JobsHistorySummary(JobDBBase): + __tablename__ = "JobsHistorySummary" + id = Column("ID", Integer, primary_key=True, autoincrement=True) + status = Column("Status", String(32)) + site = Column("Site", String(100)) + owner = Column("Owner", String(64)) + owner_group = Column("OwnerGroup", String(128)) + vo = Column("VO", String(64)) + job_group = Column("JobGroup", String(32)) + job_type = Column("JobType", String(32)) + application_status = Column("ApplicationStatus", String(255)) + minor_status = Column("MinorStatus", String(128)) + job_count = Column("JobCount", Integer, default=0) + reschedule_sum = Column("RescheduleSum", Integer, default=0) + + __table_args__ = ( + UniqueConstraint( + "Status", + "Site", + "Owner", + "OwnerGroup", # TODO: OwnerGroup(32) + "VO", + "JobGroup", + "JobType", + "ApplicationStatus", # TODO: ApplicationStatus(128) + "MinorStatus", + name="uq_summary", + ), + ) + + +########################################################### +# Triggers defined for SqlAlchemy +########################################################### +# This take place during the execution on the client, +# the MySQL database does not see this, the triggers +# are executed within the python code +########################################################### + +# trg_Jobs_insert = DDL('''\ +# CREATE TRIGGER trg_Jobs_insert +# AFTER INSERT ON Jobs +# FOR EACH ROW +# BEGIN +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType, +# ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO, +# NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus, +# NEW.MinorStatus, 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; +# END;''') + +# trg_Jobs_delete = DDL('''\ +# CREATE TRIGGER trg_Jobs_delete +# AFTER DELETE ON Jobs +# FOR EACH ROW +# BEGIN +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Remove zero rows +# DELETE FROM JobsHistorySummary +# WHERE JobCount = 0 +# AND Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; +# END;''') + +# trg_Jobs_update_status = DDL('''\ +# CREATE TRIGGER trg_Jobs_update_status +# AFTER UPDATE ON Jobs +# FOR EACH ROW +# BEGIN +# IF OLD.Status != NEW.Status THEN + +# -- Decrease count from old status +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Delete row if count drops to zero +# DELETE FROM JobsHistorySummary WHERE JobCount = 0; + +# -- Increase count for new status +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, JobGroup, VO, +# JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup, +# NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus, +# 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; + +# END IF; +# END;''') + +# # Sqlalchemy trigger creation +# event.listen(Jobs.__table__, 'after_create', trg_Jobs_insert) +# event.listen(Jobs.__table__, 'after_create', trg_Jobs_delete) +# event.listen(Jobs.__table__, 'after_create', trg_Jobs_update_status) + +########################################################### +# Triggers defined for Alembic +########################################################### +# If we use Alembic, we need a class prepared for it to +# understand, using ReplaceableObjects. +# This creates the triggers at SQL level, inside the db. +########################################################### + +trg_Jobs_insert = Trigger( + name="trg_Jobs_insert", + when="AFTER", + action="INSERT", + table="Jobs", + time="FOR EACH ROW", + body="""\ + INSERT INTO JobsHistorySummary ( + Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType, + ApplicationStatus, MinorStatus, JobCount, RescheduleSum + ) + VALUES ( + NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO, + NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus, + NEW.MinorStatus, 1, NEW.RescheduleCounter + ) + ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, + RescheduleSum = RescheduleSum + NEW.RescheduleCounter;""", +) + +trg_Jobs_delete = Trigger( + name="trg_Jobs_delete", + when="AFTER", + action="DELETE", + table="Jobs", + time="FOR EACH ROW", + body="""\ + UPDATE JobsHistorySummary + SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter + WHERE Status = OLD.Status + AND Site = OLD.Site + AND Owner = OLD.Owner + AND OwnerGroup = OLD.OwnerGroup + AND VO = OLD.VO + AND JobGroup = OLD.JobGroup + AND JobType = OLD.JobType + AND ApplicationStatus = OLD.ApplicationStatus + AND MinorStatus = OLD.MinorStatus; + + -- Remove zero rows + DELETE FROM JobsHistorySummary + WHERE JobCount = 0 + AND Status = OLD.Status + AND Site = OLD.Site + AND Owner = OLD.Owner + AND OwnerGroup = OLD.OwnerGroup + AND VO = OLD.VO + AND JobGroup = OLD.JobGroup + AND JobType = OLD.JobType + AND ApplicationStatus = OLD.ApplicationStatus + AND MinorStatus = OLD.MinorStatus;""", +) + +trg_Jobs_update = Trigger( + name="trg_Jobs_update_status", + when="AFTER", + action="UPDATE", + table="Jobs", + time="FOR EACH ROW", + body="""\ + IF OLD.Status != NEW.Status THEN + + -- Decrease count from old status + UPDATE JobsHistorySummary + SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter + WHERE Status = OLD.Status + AND Site = OLD.Site + AND Owner = OLD.Owner + AND OwnerGroup = OLD.OwnerGroup + AND VO = OLD.VO + AND JobGroup = OLD.JobGroup + AND JobType = OLD.JobType + AND ApplicationStatus = OLD.ApplicationStatus + AND MinorStatus = OLD.MinorStatus; + + -- Delete row if count drops to zero + DELETE FROM JobsHistorySummary WHERE JobCount = 0; + + -- Increase count for new status + INSERT INTO JobsHistorySummary ( + Status, Site, Owner, OwnerGroup, JobGroup, VO, + JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum + ) + VALUES ( + NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup, + NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus, + 1, NEW.RescheduleCounter + ) + ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, + RescheduleSum = RescheduleSum + NEW.RescheduleCounter; + + END IF;""", +) + +trg_Jobs_insert.register_trigger(JobDBBase.metadata) +trg_Jobs_delete.register_trigger(JobDBBase.metadata) +trg_Jobs_update.register_trigger(JobDBBase.metadata) diff --git a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py index bff7c460c..fe80b9ee6 100644 --- a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py +++ b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py @@ -26,7 +26,7 @@ class PilotAgents(PilotAgentsDBBase): destination_site = Column("DestinationSite", String(128), default="NotAssigned") queue = Column("Queue", String(128), default="Unknown") grid_site = Column("GridSite", String(128), default="Unknown") - vo = Column("VO", String(128)) + vo = Column("VO", String(64)) grid_type = Column("GridType", String(32), default="LCG") benchmark = Column("BenchMark", Double, default=0.0) submission_time = NullColumn("SubmissionTime", DateTime) @@ -58,3 +58,76 @@ class PilotOutput(PilotAgentsDBBase): pilot_id = Column("PilotID", Integer, primary_key=True) std_output = Column("StdOutput", Text) std_error = Column("StdError", Text) + + +class PilotsHistorySummary(PilotAgentsDBBase): + __tablename__ = "PilotsHistorySummary" + grid_site = Column("GridSite", String(128), primary_key=True) + destination_site = Column("DestinationSite", String(128), primary_key=True) + status = Column("Status", String(32), primary_key=True) + vo = Column("VO", String(64), primary_key=True) + pilot_count = Column("PilotCount", Integer, default=0) + + +# TODO: + +# DELIMITER // + +# CREATE TRIGGER trg_PilotAgents_insert +# AFTER INSERT ON PilotAgents +# FOR EACH ROW +# BEGIN +# INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount) +# VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1) +# ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1; +# END; +# // + +# CREATE TRIGGER trg_PilotAgents_delete +# AFTER DELETE ON PilotAgents +# FOR EACH ROW +# BEGIN +# UPDATE PilotsHistorySummary +# SET PilotCount = PilotCount - 1 +# WHERE GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; + +# -- Remove zero rows +# DELETE FROM PilotsHistorySummary +# WHERE PilotCount = 0 +# AND GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; +# END; +# // + +# CREATE TRIGGER trg_PilotAgents_update_status +# AFTER UPDATE ON PilotAgents +# FOR EACH ROW +# BEGIN +# IF OLD.Status != NEW.Status THEN + +# -- Decrease count from old status +# UPDATE PilotsHistorySummary +# SET PilotCount = PilotCount - 1 +# WHERE GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; + +# -- Delete row if count drops to zero +# DELETE FROM PilotsHistorySummary WHERE PilotCount = 0; + +# -- Increase count for new status +# INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount) +# VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1) +# ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1; + +# END IF; +# END; +# // + +# DELIMITER ; diff --git a/diracx-db/src/diracx/db/sql/utils/__init__.py b/diracx-db/src/diracx/db/sql/utils/__init__.py index 69b78b4bf..757913dfc 100644 --- a/diracx-db/src/diracx/db/sql/utils/__init__.py +++ b/diracx-db/src/diracx/db/sql/utils/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from .alembic import Trigger from .base import ( BaseSQLDB, SQLDBUnavailableError, @@ -22,4 +23,5 @@ "substract_date", "hash", "SQLDBUnavailableError", + "Trigger", ) diff --git a/diracx-db/src/diracx/db/sql/utils/alembic.py b/diracx-db/src/diracx/db/sql/utils/alembic.py new file mode 100644 index 000000000..c53832508 --- /dev/null +++ b/diracx-db/src/diracx/db/sql/utils/alembic.py @@ -0,0 +1,198 @@ +"""Alembic Utilities.""" + +from __future__ import annotations + +from alembic.autogenerate import comparators, renderers +from alembic.operations import MigrateOperation, Operations +from sqlalchemy import text + +# Typing purposes +from sqlalchemy.sql.schema import MetaData + + +class Trigger: + """Creates trigger definitions in a format accepted by Alembic.""" + + def __init__( + self, name: str, when: str, action: str, table: str, time: str, body: str + ): + self.name: str = name + self.when: str = when + self.action: str = action + self.table: str = table + self.time: str = time + self.body: str = body + + def create(self): + return f"""\ + CREATE TRIGGER {self.name} + {self.when} {self.action} + ON {self.table} + {self.time} + BEGIN + {self.body} + END""" + + def drop(self): + return f"DROP TRIGGER {self.name}" + + def __eq__(self, other: Trigger): + return self.name == other.name + + def __repr__(self): + return "Trigger(name=%r, when=%r, action=%r, table=%r, time=%r, body=%r)" % ( + self.name, + self.when, + self.action, + self.table, + self.time, + self.body, + ) + + def register_trigger(self, metadata: MetaData): + """Add the trigger to database metadata.""" + metadata.info.setdefault("triggers", list()).append(self) + + +# Custom Alemic Operations +########################## +@Operations.register_operation("create_trigger") +class CreateTriggerOp(MigrateOperation): + """Defines the operations to create triggers + Executed by calling op.create_trigger. + """ + + def __init__(self, trigger: Trigger): + self.trigger = trigger + + @classmethod + def create_trigger(cls, operations, name: str, **kw): + """Issue a "CREATE TRIGGER" instruction.""" + op = CreateTriggerOp(name, **kw) + return operations.invoke(op) + + def reverse(self): + # only needed to support autogenerate + return DropTriggerOp(self.trigger) + + +@Operations.register_operation("drop_trigger") +class DropTriggerOp(MigrateOperation): + """Defines the operations to drop triggers + Executed by calling op.drop_trigger. + """ + + def __init__(self, trigger: Trigger): + self.trigger = trigger + + @classmethod + def drop_trigger(cls, operations, name: str, **kw): + """Issue a "DROP TRIGGER" instruction.""" + op = DropTriggerOp(name) + return operations.invoke(op) + + def reverse(self): + # only needed to support autogenerate + return CreateTriggerOp(self.trigger) + + +@Operations.implementation_for(CreateTriggerOp) +def create_trigger(operations, operation): + """Receives a CreteTriggerOp operation and executes its sql text for its creation.""" + sql_text = operation.trigger.create() + operations.execute(sql_text) + + +@Operations.implementation_for(DropTriggerOp) +def drop_trigger(operations, operation): + """Receives a DropTriggerOp operation and executes its sql text for its removal.""" + sql_text = operation.trigger.drop() + operations.execute(sql_text) + + +# +# This function tells Alembic how to compare the state of the sqlalchemy metadata +# to the one found in the currently deployed database +# +# Due to triggers being stored in the database metadata, and not inside the table, the comparator +# executes at schema level. +@comparators.dispatch_for("schema") +def compare_triggers(autogen_context, operations, schemas): + """Compares the current state of the Metadata with the previous one found in the DB.""" + all_conn_triggers = list() + + # Get the collection of Triggers objects stored inside the metadata context + metadata_triggers = autogen_context.metadata.info.setdefault("triggers", list()) + + for schema in schemas: + # !!! This SQL Statement is MySQL specific !!!! + statement = text( + f"""SELECT TRIGGER_NAME, EVENT_MANIPULATION, ACTION_ORIENTATION, ACTION_TIMING, ACTION_STATEMENT \ + FROM information_schema.triggers \ + WHERE information_schema.triggers.trigger_schema \ + LIKE '{schema}';""" + ) + + for row in autogen_context.connection.execute(statement): + trigger = Trigger( + name=row["TRIGGER_NAME"], + when=row["ACTION_TIMING"], + action=row["EVENT_MANIPULATION"], + table=schema, + time="FOR EACH ROW", + body=row["ACTION_STATEMENT"], + ) + + all_conn_triggers.append(trigger) + + # For new triggers found in the metadata + for trigger in metadata_triggers: + # The trigger cannot be already in the db + if trigger in all_conn_triggers: + continue + + # The trigger is new, so produce a CreateTriggerOp directive + operations.ops.append(CreateTriggerOp(trigger)) + + # For triggers that are in the database + for trigger in all_conn_triggers: + # The trigger cannot be in the metadata + if trigger in metadata_triggers: + continue + + # The trigger got removed, so produce a DropTriggerOp directives + operations.ops.append(DropTriggerOp(trigger)) + + +# +# The renderer functions let alembic produce text that will be created inside the +# upgrade or downgrade action functions. +# +# This renderers also save some information inside an special dictionary called mutable_structure +# which let's us produce code inside the ".mako" template file +@renderers.dispatch_for(CreateTriggerOp) +def render_create_sequence(autogen_context, op: CreateTriggerOp): + """Almebic code renderer for CreateTrigger operations.""" + trigger = op.trigger + + ctx = autogen_context.opts["template_args"]["mutable_structure"]["triggers"] + + if trigger not in ctx: + ctx.append(trigger) + + # This part ends up being inside alembic's updagrade() or downgrade() functions + return f"op.create_trigger({trigger.name})" + + +@renderers.dispatch_for(DropTriggerOp) +def render_drop_sequence(autogen_context, op: DropTriggerOp): + """Almebic code renderer for DropTrigger operations.""" + trigger = op.trigger + + ctx = autogen_context.opts["template_args"]["mutable_structure"]["triggers"] + + if trigger not in ctx: + ctx.append(trigger) + + # This part ends up being inside alembic's updagrade() or downgrade() functions + return f"op.drop_trigger({trigger.name})"