From b5cdfc3d0ebb03213e9b71e79a0ad3f8d7598cbd Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 27 May 2025 11:00:29 +0200 Subject: [PATCH 1/2] fix: VO is VARCHAR(64) --- diracx-db/src/diracx/db/sql/job/schema.py | 4 ++-- diracx-db/src/diracx/db/sql/pilot_agents/schema.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index bb9f60bf1..23cce44f9 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -62,7 +62,7 @@ class Jobs(JobDBBase): job_name = Column("JobName", String(128), default="Unknown") owner = Column("Owner", String(64), default="Unknown") owner_group = Column("OwnerGroup", String(128), default="Unknown") - vo = Column("VO", String(32)) + vo = Column("VO", String(64)) submission_time = NullColumn("SubmissionTime", DateTime) reschedule_time = NullColumn("RescheduleTime", DateTime) last_update_time = NullColumn("LastUpdateTime", DateTime) @@ -153,6 +153,6 @@ class JobCommands(JobDBBase): ) command = Column("Command", String(100)) arguments = Column("Arguments", String(100)) - status = Column("Status", String(64), default="Received") + status = Column("Status", String(32), default="Received") reception_time = Column("ReceptionTime", DateTime, primary_key=True) execution_time = NullColumn("ExecutionTime", DateTime) diff --git a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py index bff7c460c..7b75c20ba 100644 --- a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py +++ b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py @@ -26,7 +26,7 @@ class PilotAgents(PilotAgentsDBBase): destination_site = Column("DestinationSite", String(128), default="NotAssigned") queue = Column("Queue", String(128), default="Unknown") grid_site = Column("GridSite", String(128), default="Unknown") - vo = Column("VO", String(128)) + vo = Column("VO", String(64)) grid_type = Column("GridType", String(32), default="LCG") benchmark = Column("BenchMark", Double, default=0.0) submission_time = NullColumn("SubmissionTime", DateTime) From ab08fe49694f058722b0418abf17e08b03dc1c74 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 2 Jun 2025 16:29:37 +0200 Subject: [PATCH 2/2] feat: define summary tables --- diracx-db/src/diracx/db/sql/job/schema.py | 127 ++++++++++++++++++ .../src/diracx/db/sql/pilot_agents/schema.py | 73 ++++++++++ 2 files changed, 200 insertions(+) diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index 23cce44f9..2e20aa3a8 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -8,6 +8,7 @@ Integer, String, Text, + UniqueConstraint, ) from sqlalchemy.orm import declarative_base @@ -156,3 +157,129 @@ class JobCommands(JobDBBase): status = Column("Status", String(32), default="Received") reception_time = Column("ReceptionTime", DateTime, primary_key=True) execution_time = NullColumn("ExecutionTime", DateTime) + + +class JobsHistorySummary(JobDBBase): + __tablename__ = "JobsHistorySummary" + id = Column("ID", Integer, primary_key=True, autoincrement=True) + status = Column("Status", String(32)) + site = Column("Site", String(100)) + owner = Column("Owner", String(64)) + owner_group = Column("OwnerGroup", String(128)) + vo = Column("VO", String(64)) + job_group = Column("JobGroup", String(32)) + job_type = Column("JobType", String(32)) + application_status = Column("ApplicationStatus", String(255)) + minor_status = Column("MinorStatus", String(128)) + job_count = Column("JobCount", Integer, default=0) + reschedule_sum = Column("RescheduleSum", Integer, default=0) + + __table_args__ = ( + UniqueConstraint( + "Status", + "Site", + "Owner", + "OwnerGroup", # TODO: OwnerGroup(32) + "VO", + "JobGroup", + "JobType", + "ApplicationStatus", # TODO: ApplicationStatus(128) + "MinorStatus", + name="uq_summary", + ), + ) + + +# TODO: + +# DELIMITER // + +# CREATE TRIGGER trg_Jobs_insert +# AFTER INSERT ON Jobs +# FOR EACH ROW +# BEGIN +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, VO, JobGroup, JobType, +# ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.VO, +# NEW.JobGroup, NEW.JobType, NEW.ApplicationStatus, +# NEW.MinorStatus, 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; +# END; +# // + +# CREATE TRIGGER trg_Jobs_delete +# AFTER DELETE ON Jobs +# FOR EACH ROW +# BEGIN +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Remove zero rows +# DELETE FROM JobsHistorySummary +# WHERE JobCount = 0 +# AND Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; +# END; +# // + +# CREATE TRIGGER trg_Jobs_update_status +# AFTER UPDATE ON Jobs +# FOR EACH ROW +# BEGIN +# IF OLD.Status != NEW.Status THEN + +# -- Decrease count from old status +# UPDATE JobsHistorySummary +# SET JobCount = JobCount - 1, RescheduleSum = RescheduleSum - OLD.RescheduleCounter +# WHERE Status = OLD.Status +# AND Site = OLD.Site +# AND Owner = OLD.Owner +# AND OwnerGroup = OLD.OwnerGroup +# AND VO = OLD.VO +# AND JobGroup = OLD.JobGroup +# AND JobType = OLD.JobType +# AND ApplicationStatus = OLD.ApplicationStatus +# AND MinorStatus = OLD.MinorStatus; + +# -- Delete row if count drops to zero +# DELETE FROM JobsHistorySummary WHERE JobCount = 0; + +# -- Increase count for new status +# INSERT INTO JobsHistorySummary ( +# Status, Site, Owner, OwnerGroup, JobGroup, VO, +# JobType, ApplicationStatus, MinorStatus, JobCount, RescheduleSum +# ) +# VALUES ( +# NEW.Status, NEW.Site, NEW.Owner, NEW.OwnerGroup, NEW.JobGroup, +# NEW.VO, NEW.JobType, NEW.ApplicationStatus, NEW.MinorStatus, +# 1, NEW.RescheduleCounter +# ) +# ON DUPLICATE KEY UPDATE JobCount = JobCount + 1, +# RescheduleSum = RescheduleSum + NEW.RescheduleCounter; + +# END IF; +# END; +# // + +# DELIMITER ; diff --git a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py index 7b75c20ba..fe80b9ee6 100644 --- a/diracx-db/src/diracx/db/sql/pilot_agents/schema.py +++ b/diracx-db/src/diracx/db/sql/pilot_agents/schema.py @@ -58,3 +58,76 @@ class PilotOutput(PilotAgentsDBBase): pilot_id = Column("PilotID", Integer, primary_key=True) std_output = Column("StdOutput", Text) std_error = Column("StdError", Text) + + +class PilotsHistorySummary(PilotAgentsDBBase): + __tablename__ = "PilotsHistorySummary" + grid_site = Column("GridSite", String(128), primary_key=True) + destination_site = Column("DestinationSite", String(128), primary_key=True) + status = Column("Status", String(32), primary_key=True) + vo = Column("VO", String(64), primary_key=True) + pilot_count = Column("PilotCount", Integer, default=0) + + +# TODO: + +# DELIMITER // + +# CREATE TRIGGER trg_PilotAgents_insert +# AFTER INSERT ON PilotAgents +# FOR EACH ROW +# BEGIN +# INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount) +# VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1) +# ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1; +# END; +# // + +# CREATE TRIGGER trg_PilotAgents_delete +# AFTER DELETE ON PilotAgents +# FOR EACH ROW +# BEGIN +# UPDATE PilotsHistorySummary +# SET PilotCount = PilotCount - 1 +# WHERE GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; + +# -- Remove zero rows +# DELETE FROM PilotsHistorySummary +# WHERE PilotCount = 0 +# AND GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; +# END; +# // + +# CREATE TRIGGER trg_PilotAgents_update_status +# AFTER UPDATE ON PilotAgents +# FOR EACH ROW +# BEGIN +# IF OLD.Status != NEW.Status THEN + +# -- Decrease count from old status +# UPDATE PilotsHistorySummary +# SET PilotCount = PilotCount - 1 +# WHERE GridSite = OLD.GridSite +# AND DestinationSite = OLD.DestinationSite +# AND Status = OLD.Status +# AND VO = OLD.VO; + +# -- Delete row if count drops to zero +# DELETE FROM PilotsHistorySummary WHERE PilotCount = 0; + +# -- Increase count for new status +# INSERT INTO PilotsHistorySummary (GridSite, DestinationSite, Status, VO, PilotCount) +# VALUES (NEW.GridSite, NEW.DestinationSite, NEW.Status, NEW.VO, 1) +# ON DUPLICATE KEY UPDATE PilotCount = PilotCount + 1; + +# END IF; +# END; +# // + +# DELIMITER ;