From 55c9d21ff6e5aaaf0aa0548374987a0e616469cb Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Thu, 23 Oct 2025 15:25:52 +0200 Subject: [PATCH 1/9] feat: verify CS config using diracx model validation --- .../ConfigurationSystem/private/DiracX.py | 21 +++++++++++++++++++ .../private/Modificator.py | 12 ++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 src/DIRAC/ConfigurationSystem/private/DiracX.py diff --git a/src/DIRAC/ConfigurationSystem/private/DiracX.py b/src/DIRAC/ConfigurationSystem/private/DiracX.py new file mode 100644 index 00000000000..2810a3823eb --- /dev/null +++ b/src/DIRAC/ConfigurationSystem/private/DiracX.py @@ -0,0 +1,21 @@ +from diracx.core.config.schema import Config as DiracxConfig +from pydantic import ValidationError + +from DIRAC import S_ERROR, S_OK + + +def diracxVerifyConfig(cfgData): + """Verify CS config using DiracX config validation + + Args: + cfgData: CS config data + + Returns: + S_OK | S_ERROR: Value: diracx config validation + """ + cfg = cfgData.getAsDict() + try: + validation = DiracxConfig.model_validate(cfg) + except ValidationError as exc: + return S_ERROR(exc) + return S_OK(validation) diff --git a/src/DIRAC/ConfigurationSystem/private/Modificator.py b/src/DIRAC/ConfigurationSystem/private/Modificator.py index 04f8afc70e4..74cf0077f13 100755 --- a/src/DIRAC/ConfigurationSystem/private/Modificator.py +++ b/src/DIRAC/ConfigurationSystem/private/Modificator.py @@ -1,13 +1,16 @@ """ This is the guy that actually modifies the content of the CS """ -import zlib -import difflib import datetime +import difflib +import zlib from diraccfg import CFG -from DIRAC.Core.Utilities import List + +from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.private.DiracX import diracxVerifyConfig from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.Core.Utilities import List class Modificator: @@ -239,6 +242,9 @@ def __str__(self): return str(self.cfgData) def commit(self): + resVerif = diracxVerifyConfig(self.cfgData) + if not resVerif["OK"]: + gLogger.warn(resVerif["Value"]) compressedData = zlib.compress(str(self.cfgData).encode(), 9) return self.rpcClient.commitNewData(compressedData) From f1f0b812ae75bf90655195a43a21862856e21103 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Thu, 23 Oct 2025 15:45:30 +0200 Subject: [PATCH 2/9] fix: s_error message instead of value --- src/DIRAC/ConfigurationSystem/private/Modificator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/ConfigurationSystem/private/Modificator.py b/src/DIRAC/ConfigurationSystem/private/Modificator.py index 74cf0077f13..27bad889341 100755 --- a/src/DIRAC/ConfigurationSystem/private/Modificator.py +++ b/src/DIRAC/ConfigurationSystem/private/Modificator.py @@ -244,7 +244,7 @@ def __str__(self): def commit(self): resVerif = diracxVerifyConfig(self.cfgData) if not resVerif["OK"]: - gLogger.warn(resVerif["Value"]) + gLogger.warn(resVerif["Message"]) compressedData = zlib.compress(str(self.cfgData).encode(), 9) return self.rpcClient.commitNewData(compressedData) From 8476312a8bfaf278297613cb79b837c8f344c77f Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Fri, 24 Oct 2025 09:30:12 +0200 Subject: [PATCH 3/9] refactor: move diracx function in Framework --- .../ConfigurationSystem/private/DiracX.py | 21 ----------- .../private/Modificator.py | 2 +- src/DIRAC/FrameworkSystem/Utilities/diracx.py | 37 ++++++++++++++----- 3 files changed, 28 insertions(+), 32 deletions(-) delete mode 100644 src/DIRAC/ConfigurationSystem/private/DiracX.py diff --git a/src/DIRAC/ConfigurationSystem/private/DiracX.py b/src/DIRAC/ConfigurationSystem/private/DiracX.py deleted file mode 100644 index 2810a3823eb..00000000000 --- a/src/DIRAC/ConfigurationSystem/private/DiracX.py +++ /dev/null @@ -1,21 +0,0 @@ -from diracx.core.config.schema import Config as DiracxConfig -from pydantic import ValidationError - -from DIRAC import S_ERROR, S_OK - - -def diracxVerifyConfig(cfgData): - """Verify CS config using DiracX config validation - - Args: - cfgData: CS config data - - Returns: - S_OK | S_ERROR: Value: diracx config validation - """ - cfg = cfgData.getAsDict() - try: - validation = DiracxConfig.model_validate(cfg) - except ValidationError as exc: - return S_ERROR(exc) - return S_OK(validation) diff --git a/src/DIRAC/ConfigurationSystem/private/Modificator.py b/src/DIRAC/ConfigurationSystem/private/Modificator.py index 27bad889341..f2030e152da 100755 --- a/src/DIRAC/ConfigurationSystem/private/Modificator.py +++ b/src/DIRAC/ConfigurationSystem/private/Modificator.py @@ -8,9 +8,9 @@ from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.private.DiracX import diracxVerifyConfig from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Utilities import List +from DIRAC.FrameworkSystem.Utilities.diracx import diracxVerifyConfig class Modificator: diff --git a/src/DIRAC/FrameworkSystem/Utilities/diracx.py b/src/DIRAC/FrameworkSystem/Utilities/diracx.py index 631e4167c3e..fca8c405863 100644 --- a/src/DIRAC/FrameworkSystem/Utilities/diracx.py +++ b/src/DIRAC/FrameworkSystem/Utilities/diracx.py @@ -1,20 +1,20 @@ -import requests - -from cachetools import TTLCache, LRUCache, cached -from cachetools.keys import hashkey +from collections.abc import Generator +from contextlib import contextmanager from pathlib import Path from tempfile import NamedTemporaryFile from typing import Any -from collections.abc import Generator -from DIRAC import gConfig -from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from contextlib import contextmanager +import requests +from cachetools import LRUCache, TTLCache, cached +from cachetools.keys import hashkey +from diracx.core.config.schema import Config as DiracxConfig +from diracx.core.models import TokenResponse from diracx.core.preferences import DiracxPreferences - from diracx.core.utils import write_credentials +from pydantic import ValidationError -from diracx.core.models import TokenResponse +from DIRAC import S_ERROR, S_OK, gConfig +from DIRAC.ConfigurationSystem.Client.Helpers import Registry try: from diracx.client.sync import SyncDiracClient @@ -104,3 +104,20 @@ def TheImpersonator(credDict: dict[str, Any], *, source: str = "") -> Generator[ client.__enter__() diracx_client_cache[token_location] = client yield client + + +def diracxVerifyConfig(cfgData): + """Verify CS config using DiracX config validation + + Args: + cfgData: CFG data + + Returns: + S_OK | S_ERROR: Value: diracx Config validation + """ + cfg = cfgData.getAsDict() + try: + validation = DiracxConfig.model_validate(cfg) + except ValidationError as exc: + return S_ERROR(exc) + return S_OK(validation) From 887a8d23627cf9696d62cdd0bf7bb6312b4f979e Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Mon, 3 Nov 2025 16:51:30 +0100 Subject: [PATCH 4/9] feat: run dirac legacy cs-sync on CS commit --- .../private/Modificator.py | 4 +-- src/DIRAC/FrameworkSystem/Utilities/diracx.py | 25 ++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/DIRAC/ConfigurationSystem/private/Modificator.py b/src/DIRAC/ConfigurationSystem/private/Modificator.py index f2030e152da..b2c662dfa81 100755 --- a/src/DIRAC/ConfigurationSystem/private/Modificator.py +++ b/src/DIRAC/ConfigurationSystem/private/Modificator.py @@ -6,7 +6,7 @@ from diraccfg import CFG -from DIRAC import gLogger +from DIRAC import S_ERROR from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Utilities import List @@ -244,7 +244,7 @@ def __str__(self): def commit(self): resVerif = diracxVerifyConfig(self.cfgData) if not resVerif["OK"]: - gLogger.warn(resVerif["Message"]) + return S_ERROR(resVerif["Message"]) compressedData = zlib.compress(str(self.cfgData).encode(), 9) return self.rpcClient.commitNewData(compressedData) diff --git a/src/DIRAC/FrameworkSystem/Utilities/diracx.py b/src/DIRAC/FrameworkSystem/Utilities/diracx.py index fca8c405863..e8723585a44 100644 --- a/src/DIRAC/FrameworkSystem/Utilities/diracx.py +++ b/src/DIRAC/FrameworkSystem/Utilities/diracx.py @@ -1,12 +1,17 @@ +import os +import re +import subprocess from collections.abc import Generator from contextlib import contextmanager from pathlib import Path from tempfile import NamedTemporaryFile +import tempfile from typing import Any import requests from cachetools import LRUCache, TTLCache, cached from cachetools.keys import hashkey +from diracx.cli.internal.legacy import _apply_fixes from diracx.core.config.schema import Config as DiracxConfig from diracx.core.models import TokenResponse from diracx.core.preferences import DiracxPreferences @@ -115,9 +120,17 @@ def diracxVerifyConfig(cfgData): Returns: S_OK | S_ERROR: Value: diracx Config validation """ - cfg = cfgData.getAsDict() - try: - validation = DiracxConfig.model_validate(cfg) - except ValidationError as exc: - return S_ERROR(exc) - return S_OK(validation) + os.environ["DIRAC_COMPAT_ENABLE_CS_CONVERSION"] = "true" + with tempfile.NamedTemporaryFile() as temp_cfg: + with tempfile.NamedTemporaryFile() as temp_diracx_cfg: + cfgData.writeToFile(temp_cfg) + cmd = ["dirac", "internal", "legacy", "cs-sync", temp_cfg, temp_diracx_cfg] + res = subprocess.run(cmd, capture_output=True, text=True, timeout=15) + os.environ.pop("DIRAC_COMPAT_ENABLE_CS_CONVERSION") + if res.returncode == 0: + return S_OK(res.stdout) + else: + err = res.stderr.strip() + match = re.search(r"(ValidationError:.*)", err, flags=re.DOTALL) + if match: + return S_ERROR(match.group(1)) From 674ad5daa8fd5d04b104b9f29800c98e67855fad Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Tue, 4 Nov 2025 12:07:21 +0100 Subject: [PATCH 5/9] feat: verify if there is the proper option in the CS --- src/DIRAC/ConfigurationSystem/private/Modificator.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/DIRAC/ConfigurationSystem/private/Modificator.py b/src/DIRAC/ConfigurationSystem/private/Modificator.py index b2c662dfa81..6b1046fac78 100755 --- a/src/DIRAC/ConfigurationSystem/private/Modificator.py +++ b/src/DIRAC/ConfigurationSystem/private/Modificator.py @@ -242,9 +242,13 @@ def __str__(self): return str(self.cfgData) def commit(self): - resVerif = diracxVerifyConfig(self.cfgData) - if not resVerif["OK"]: - return S_ERROR(resVerif["Message"]) + retOpt = gConfigurationData.extractOptionFromCFG( + "/Systems/Configuration/Services/Server/VerifyDiracXSyncOnCommit" + ) + if retOpt == "True": + resVerif = diracxVerifyConfig(self.cfgData) + if not resVerif["OK"]: + return S_ERROR(resVerif["Message"]) compressedData = zlib.compress(str(self.cfgData).encode(), 9) return self.rpcClient.commitNewData(compressedData) From f67d84e63d6dc7b6b6243cf018b3603347f4d0ed Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Tue, 4 Nov 2025 16:37:29 +0100 Subject: [PATCH 6/9] refactor: use getValue --- src/DIRAC/ConfigurationSystem/private/Modificator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/DIRAC/ConfigurationSystem/private/Modificator.py b/src/DIRAC/ConfigurationSystem/private/Modificator.py index 6b1046fac78..7f07a506075 100755 --- a/src/DIRAC/ConfigurationSystem/private/Modificator.py +++ b/src/DIRAC/ConfigurationSystem/private/Modificator.py @@ -242,9 +242,7 @@ def __str__(self): return str(self.cfgData) def commit(self): - retOpt = gConfigurationData.extractOptionFromCFG( - "/Systems/Configuration/Services/Server/VerifyDiracXSyncOnCommit" - ) + retOpt = self.getValue("/Systems/Configuration/Services/Server/VerifyDiracXSyncOnCommit") if retOpt == "True": resVerif = diracxVerifyConfig(self.cfgData) if not resVerif["OK"]: From 4c7b46a264c64fe688f4184071c588a6b4d85a0a Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Tue, 4 Nov 2025 16:38:08 +0100 Subject: [PATCH 7/9] fix: properly return error if no match --- src/DIRAC/FrameworkSystem/Utilities/diracx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DIRAC/FrameworkSystem/Utilities/diracx.py b/src/DIRAC/FrameworkSystem/Utilities/diracx.py index e8723585a44..fd01a886f8a 100644 --- a/src/DIRAC/FrameworkSystem/Utilities/diracx.py +++ b/src/DIRAC/FrameworkSystem/Utilities/diracx.py @@ -134,3 +134,4 @@ def diracxVerifyConfig(cfgData): match = re.search(r"(ValidationError:.*)", err, flags=re.DOTALL) if match: return S_ERROR(match.group(1)) + return S_ERROR(err) From 8036c6e272de13b1f72ab0b3f14ee5edc6b7cbd7 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 5 Nov 2025 16:59:52 +0100 Subject: [PATCH 8/9] fix: use temp file name instead of namedTempFile --- src/DIRAC/FrameworkSystem/Utilities/diracx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DIRAC/FrameworkSystem/Utilities/diracx.py b/src/DIRAC/FrameworkSystem/Utilities/diracx.py index fd01a886f8a..766ed48b95d 100644 --- a/src/DIRAC/FrameworkSystem/Utilities/diracx.py +++ b/src/DIRAC/FrameworkSystem/Utilities/diracx.py @@ -123,8 +123,8 @@ def diracxVerifyConfig(cfgData): os.environ["DIRAC_COMPAT_ENABLE_CS_CONVERSION"] = "true" with tempfile.NamedTemporaryFile() as temp_cfg: with tempfile.NamedTemporaryFile() as temp_diracx_cfg: - cfgData.writeToFile(temp_cfg) - cmd = ["dirac", "internal", "legacy", "cs-sync", temp_cfg, temp_diracx_cfg] + cfgData.writeToFile(temp_cfg.name) + cmd = ["dirac", "internal", "legacy", "cs-sync", temp_cfg.name, temp_diracx_cfg.name] res = subprocess.run(cmd, capture_output=True, text=True, timeout=15) os.environ.pop("DIRAC_COMPAT_ENABLE_CS_CONVERSION") if res.returncode == 0: From b3d302c9138062d3a206e9a9fcdb3b2e9560d8b8 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Fri, 21 Nov 2025 10:05:43 +0100 Subject: [PATCH 9/9] docs: add option and comment in default CS cfg --- src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg b/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg index 5776ba856db..b0d1135b6bc 100644 --- a/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg +++ b/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg @@ -6,6 +6,9 @@ Services { HandlerPath = DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py Port = 9135 + # Set to True to make a DiracX model validation on commits + # If validation fails, commit will also failed + VerifyDiracXSyncOnCommit = False # Subsection to configure authorization over the service Authorization {