refactor: restructure DIRACCommon to preserve DIRAC directory hierarchy

chrisburr · claude · chrisburr · commit 8e4479afd91b · 2025-09-01T16:52:46.000+02:00
Reorganize DIRACCommon modules to match the original DIRAC structure: - Core/Utilities/ClassAd/ClassAdLight.py (utilities belong in Core) - WorkloadManagementSystem/DB/JobDBUtils.py (preserves original location) - WorkloadManagementSystem/Utilities/ParametricJob.py (preserves original location) Update all imports across DIRAC, diracx, and DIRACCommon to use the new structured paths. Reorganize tests to match the new directory structure. This maintains the logical organization while enabling proper code migration. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
diff --git a/dirac-common/README.md b/dirac-common/README.md
@@ -8,8 +8,11 @@ This package solves the circular dependency issue where DiracX needs DIRAC utili
 
 ## Contents
 
-- `DIRACCommon.Utils.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
-- `DIRACCommon.Utils.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
+- `DIRACCommon.Core.Utilities.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ClassAd.ClassAdLight`: JDL parsing utilities
+- `DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils`: Job database utilities
+- `DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob`: Parametric job utilities
 
 ## Installation
 
@@ -20,7 +23,7 @@ pip install DIRACCommon
 ## Usage
 
 ```python
-from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
 
 def my_function():
     if success:
diff --git a/dirac-common/src/DIRACCommon/Core/Utilities/ClassAd/ClassAdLight.py b/dirac-common/src/DIRACCommon/Core/Utilities/ClassAd/ClassAdLight.py
@@ -292,4 +292,4 @@ def getAttributes(self) -> list[str]:
 
         :return: list of names as strings
         """
-        return list(self.contents)
+        return list(self.contents)
diff --git a/dirac-common/src/DIRACCommon/Core/Utilities/ClassAd/__init__.py b/dirac-common/src/DIRACCommon/Core/Utilities/ClassAd/__init__.py
@@ -0,0 +1 @@
+"""ClassAd utilities for DIRACCommon"""
diff --git a/dirac-common/src/DIRACCommon/Core/Utilities/DErrno.py b/dirac-common/src/DIRACCommon/Core/Utilities/DErrno.py
diff --git a/dirac-common/src/DIRACCommon/Core/Utilities/ReturnValues.py b/dirac-common/src/DIRACCommon/Core/Utilities/ReturnValues.py
@@ -14,7 +14,7 @@
 from typing import Any, Callable, cast, Generic, Literal, overload, Type, TypeVar, Union
 from typing_extensions import TypedDict, ParamSpec, NotRequired
 
-from DIRACCommon.Utils.DErrno import strerror
+from DIRACCommon.Core.Utilities.DErrno import strerror
 
 
 T = TypeVar("T")
diff --git a/dirac-common/src/DIRACCommon/Core/Utilities/__init__.py b/dirac-common/src/DIRACCommon/Core/Utilities/__init__.py
diff --git a/dirac-common/src/DIRACCommon/Core/__init__.py b/dirac-common/src/DIRACCommon/Core/__init__.py
@@ -0,0 +1 @@
+"""DIRACCommon Core utilities"""
diff --git a/dirac-common/src/DIRACCommon/Utils/ClassAd/__init__.py b/dirac-common/src/DIRACCommon/Utils/ClassAd/__init__.py
diff --git a/dirac-common/src/DIRACCommon/WorkloadManagementSystem/DB/JobDBUtils.py b/dirac-common/src/DIRACCommon/WorkloadManagementSystem/DB/JobDBUtils.py
@@ -30,4 +30,4 @@ def fixJDL(jdl: str) -> str:
     # Fix the possible lack of the brackets in the JDL
     if jdl.strip()[0].find("[") != 0:
         jdl = "[" + jdl + "]"
-    return jdl
+    return jdl
diff --git a/dirac-common/src/DIRACCommon/WorkloadManagementSystem/DB/__init__.py b/dirac-common/src/DIRACCommon/WorkloadManagementSystem/DB/__init__.py
@@ -0,0 +1 @@
+"""DIRACCommon WorkloadManagementSystem DB utilities"""
diff --git a/dirac-common/src/DIRACCommon/WorkloadManagementSystem/Utilities/ParametricJob.py b/dirac-common/src/DIRACCommon/WorkloadManagementSystem/Utilities/ParametricJob.py
@@ -6,9 +6,9 @@
 """
 import re
 
-from DIRACCommon.Utils.ClassAd.ClassAdLight import ClassAd
-from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR
-from DIRACCommon.Utils.DErrno import EWMSJDL
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.DErrno import EWMSJDL
 
 
 def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1):
@@ -176,4 +176,4 @@ def generateParametricJobs(jobClassAd):
         newJDL = newClassAd.asJDL()
         jobDescList.append(newJDL)
 
-    return S_OK(jobDescList)
+    return S_OK(jobDescList)
diff --git a/dirac-common/src/DIRACCommon/WorkloadManagementSystem/Utilities/__init__.py b/dirac-common/src/DIRACCommon/WorkloadManagementSystem/Utilities/__init__.py
@@ -0,0 +1 @@
+"""DIRACCommon WorkloadManagementSystem utilities"""
diff --git a/dirac-common/src/DIRACCommon/WorkloadManagementSystem/__init__.py b/dirac-common/src/DIRACCommon/WorkloadManagementSystem/__init__.py
@@ -0,0 +1 @@
+"""DIRACCommon WorkloadManagementSystem utilities"""
diff --git a/dirac-common/tests/Core/Utilities/ClassAd/__init__.py b/dirac-common/tests/Core/Utilities/ClassAd/__init__.py
diff --git a/dirac-common/tests/Core/Utilities/ClassAd/test_ClassAdLight.py b/dirac-common/tests/Core/Utilities/ClassAd/test_ClassAdLight.py
@@ -4,17 +4,17 @@
 
 import pytest
 
-from DIRACCommon.Utils.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 
 
 def test_classad_basic():
     """Test basic ClassAd functionality"""
     # Arrange
-    jdl = "[Executable = \"my_executable\"; JobName = \"Test\"]"
-    
+    jdl = '[Executable = "my_executable"; JobName = "Test"]'
+
     # Act
     classad = ClassAd(jdl)
-    
+
     # Assert
     assert classad.isOK()
     assert classad.getAttributeString("Executable") == "my_executable"
@@ -26,12 +26,12 @@ def test_classad_insert_attributes():
     # Arrange
     jdl = "[]"
     classad = ClassAd(jdl)
-    
+
     # Act
     classad.insertAttributeString("Executable", "test_exe")
     classad.insertAttributeInt("Priority", 5)
     classad.insertAttributeBool("Test", True)
-    
+
     # Assert
     assert classad.getAttributeString("Executable") == "test_exe"
     assert classad.getAttributeInt("Priority") == 5
@@ -41,12 +41,12 @@ def test_classad_insert_attributes():
 def test_classad_list_attributes():
     """Test list attributes in ClassAd"""
     # Arrange
-    jdl = "[InputData = {\"file1.txt\", \"file2.txt\"}]"
+    jdl = '[InputData = {"file1.txt", "file2.txt"}]'
     classad = ClassAd(jdl)
-    
+
     # Act
     file_list = classad.getListFromExpression("InputData")
-    
+
     # Assert
     assert len(file_list) == 2
     assert "file1.txt" in file_list
@@ -56,14 +56,14 @@ def test_classad_list_attributes():
 def test_classad_as_jdl():
     """Test converting ClassAd back to JDL"""
     # Arrange
-    original_jdl = "[Executable = \"my_executable\"; JobName = \"Test\"]"
+    original_jdl = '[Executable = "my_executable"; JobName = "Test"]'
     classad = ClassAd(original_jdl)
-    
+
     # Act
     regenerated_jdl = classad.asJDL()
-    
+
     # Assert
     assert "Executable" in regenerated_jdl
     assert "my_executable" in regenerated_jdl
     assert "JobName" in regenerated_jdl
-    assert "Test" in regenerated_jdl
+    assert "Test" in regenerated_jdl
diff --git a/dirac-common/tests/Core/Utilities/__init__.py b/dirac-common/tests/Core/Utilities/__init__.py
diff --git a/dirac-common/tests/Core/Utilities/test_DErrno.py b/dirac-common/tests/Core/Utilities/test_DErrno.py
@@ -1,7 +1,7 @@
 """Tests for DErrno module"""
 
 import pytest
-from DIRACCommon.Utils import DErrno
+from DIRACCommon.Core.Utilities import DErrno
 
 
 def test_strerror():
diff --git a/dirac-common/tests/Core/Utilities/test_ReturnValues.py b/dirac-common/tests/Core/Utilities/test_ReturnValues.py
@@ -1,6 +1,12 @@
 import pytest
 
-from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR, SErrorException, convertToReturnValue, returnValueOrRaise
+from DIRACCommon.Core.Utilities.ReturnValues import (
+    S_OK,
+    S_ERROR,
+    SErrorException,
+    convertToReturnValue,
+    returnValueOrRaise,
+)
 
 
 def test_Ok():
diff --git a/dirac-common/tests/Core/__init__.py b/dirac-common/tests/Core/__init__.py
diff --git a/dirac-common/tests/Utils/__init__.py b/dirac-common/tests/Utils/__init__.py
diff --git a/dirac-common/tests/WorkloadManagementSystem/DB/__init__.py b/dirac-common/tests/WorkloadManagementSystem/DB/__init__.py
diff --git a/dirac-common/tests/WorkloadManagementSystem/DB/test_JobDBUtils.py b/dirac-common/tests/WorkloadManagementSystem/DB/test_JobDBUtils.py
@@ -5,18 +5,18 @@
 import base64
 import zlib
 
-from DIRACCommon.Utils.JobDBUtils import compressJDL, extractJDL, fixJDL
+from DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils import compressJDL, extractJDL, fixJDL
 
 
 def test_compress_extract_jdl():
     """Test JDL compression and extraction"""
     # Arrange
-    original_jdl = "[Executable = \"my_executable\"; JobName = \"Test\"]"
-    
+    original_jdl = '[Executable = "my_executable"; JobName = "Test"]'
+
     # Act
     compressed = compressJDL(original_jdl)
     extracted = extractJDL(compressed)
-    
+
     # Assert
     assert compressed != original_jdl
     assert extracted == original_jdl
@@ -26,23 +26,23 @@ def test_compress_extract_jdl():
 def test_extract_jdl_already_decompressed():
     """Test extracting JDL that's already decompressed"""
     # Arrange
-    jdl = "[Executable = \"my_executable\"; JobName = \"Test\"]"
-    
+    jdl = '[Executable = "my_executable"; JobName = "Test"]'
+
     # Act
     result = extractJDL(jdl)
-    
+
     # Assert
     assert result == jdl
 
 
 def test_extract_jdl_bytes():
     """Test extracting JDL from bytes"""
     # Arrange
-    jdl_bytes = b"[Executable = \"my_executable\"; JobName = \"Test\"]"
-    
+    jdl_bytes = b'[Executable = "my_executable"; JobName = "Test"]'
+
     # Act
     result = extractJDL(jdl_bytes)
-    
+
     # Assert
     assert result == jdl_bytes.decode()
 
@@ -51,10 +51,10 @@ def test_fix_jdl_missing_brackets():
     """Test fixing JDL missing brackets"""
     # Arrange
     jdl_no_brackets = 'Executable = "my_executable"; JobName = "Test"'
-    
+
     # Act
     fixed = fixJDL(jdl_no_brackets)
-    
+
     # Assert
     assert fixed.startswith("[")
     assert fixed.endswith("]")
@@ -65,9 +65,9 @@ def test_fix_jdl_already_bracketed():
     """Test fixing JDL that already has brackets"""
     # Arrange
     jdl = '[Executable = "my_executable"; JobName = "Test"]'
-    
+
     # Act
     fixed = fixJDL(jdl)
-    
+
     # Assert
-    assert fixed == jdl
+    assert fixed == jdl
diff --git a/dirac-common/tests/WorkloadManagementSystem/Utilities/__init__.py b/dirac-common/tests/WorkloadManagementSystem/Utilities/__init__.py
diff --git a/dirac-common/tests/WorkloadManagementSystem/Utilities/test_ParametricJob.py b/dirac-common/tests/WorkloadManagementSystem/Utilities/test_ParametricJob.py
@@ -4,8 +4,11 @@
 
 import pytest
 
-from DIRACCommon.Utils.ParametricJob import generateParametricJobs, getParameterVectorLength
-from DIRACCommon.Utils.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob import (
+    generateParametricJobs,
+    getParameterVectorLength,
+)
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 
 TEST_JDL_NO_PARAMETERS = """
 [
@@ -135,4 +138,4 @@ def test_generateParametricJobs(jdl: str, expectedArguments: list[str]):
     for i in range(len(jobDescList)):
         jobDescription = ClassAd(jobDescList[i])
         assert jobDescription.getAttributeString("JobName") == f"Test_{i}"
-        assert jobDescription.getAttributeString("Arguments") == expectedArguments[i]
+        assert jobDescription.getAttributeString("Arguments") == expectedArguments[i]
diff --git a/dirac-common/tests/WorkloadManagementSystem/__init__.py b/dirac-common/tests/WorkloadManagementSystem/__init__.py
diff --git a/src/DIRAC/Core/Utilities/ClassAd/ClassAdLight.py b/src/DIRAC/Core/Utilities/ClassAd/ClassAdLight.py
@@ -3,7 +3,7 @@
 """
 
 # Import from DIRACCommon for backward compatibility
-from DIRACCommon.Utils.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 
 # Re-export for backward compatibility
-__all__ = ["ClassAd"]
+__all__ = ["ClassAd"]
diff --git a/src/DIRAC/Core/Utilities/DErrno.py b/src/DIRAC/Core/Utilities/DErrno.py
@@ -41,7 +41,7 @@
 import sys
 
 # Import all the stateless parts from DIRACCommon
-from DIRACCommon.Utils.DErrno import *  # noqa: F401, F403
+from DIRACCommon.Core.Utilities.DErrno import *  # noqa: F401, F403
 
 from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 
diff --git a/src/DIRAC/Core/Utilities/ReturnValues.py b/src/DIRAC/Core/Utilities/ReturnValues.py
@@ -1,10 +1,10 @@
 """Backward compatibility wrapper - moved to DIRACCommon
 
-This module has been moved to DIRACCommon.Utils.ReturnValues to avoid
+This module has been moved to DIRACCommon.Core.Utilities.ReturnValues to avoid
 circular dependencies and allow DiracX to use these utilities without
 triggering DIRAC's global state initialization.
 
 All exports are maintained for backward compatibility.
 """
 # Re-export everything from DIRACCommon for backward compatibility
-from DIRACCommon.Utils.ReturnValues import *  # noqa: F401, F403
+from DIRACCommon.Core.Utilities.ReturnValues import *  # noqa: F401, F403
diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py
@@ -11,7 +11,7 @@
 from DIRAC.WorkloadManagementSystem.Client.JobState.JobManifest import JobManifest
 
 # Import stateless functions from DIRACCommon for backward compatibility
-from DIRACCommon.Utils.JobDBUtils import compressJDL, extractJDL, fixJDL
+from DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils import compressJDL, extractJDL, fixJDL
 
 getDIRACPlatform = returnValueOrRaise(
     ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", "getDIRACPlatform")
@@ -32,8 +32,6 @@ def checkAndAddOwner(jdl: str, owner: str, ownerGroup: str) -> JobManifest:
     return S_OK(jobManifest)
 
 
-
-
 def checkAndPrepareJob(jobID, classAdJob, classAdReq, owner, ownerGroup, jobAttrs, vo):
     error = ""
 
diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py b/src/DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py
@@ -6,10 +6,10 @@
 """
 
 # Import from DIRACCommon for backward compatibility
-from DIRACCommon.Utils.ParametricJob import (
+from DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob import (
     getParameterVectorLength,
     generateParametricJobs,
 )
 
 # Re-export for backward compatibility
-__all__ = ["getParameterVectorLength", "generateParametricJobs"]
+__all__ = ["getParameterVectorLength", "generateParametricJobs"]
diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py
@@ -19,12 +19,12 @@ def test_backward_compatibility_import():
         Parameters = { "a", "b", "c" }
     ]
     """
-    
+
     # Act - Test that we can import and use the functions from DIRAC
     jobDescription = ClassAd(jdl)
     vector_result = getParameterVectorLength(jobDescription)
     generate_result = generateParametricJobs(jobDescription)
-    
+
     # Assert - Verify functions work correctly
     assert vector_result["OK"]
     assert vector_result["Value"] == 3
@@ -35,21 +35,21 @@ def test_backward_compatibility_import():
 # Import and run the comprehensive tests from DIRACCommon to avoid duplication
 # This ensures the DIRAC re-exports work with the full test suite
 try:
-    from DIRACCommon.tests.Utils.test_ParametricJob import (
+    from DIRACCommon.tests.WorkloadManagementSystem.Utilities.test_ParametricJob import (
         test_getParameterVectorLength_successful,
-        test_getParameterVectorLength_unsuccessful, 
+        test_getParameterVectorLength_unsuccessful,
         test_generateParametricJobs,
     )
-    
+
     # Re-export the DIRACCommon tests so they run as part of DIRAC test suite
     # This validates that the backward compatibility imports work correctly
     __all__ = [
-        'test_backward_compatibility_import',
-        'test_getParameterVectorLength_successful',
-        'test_getParameterVectorLength_unsuccessful',
-        'test_generateParametricJobs',
+        "test_backward_compatibility_import",
+        "test_getParameterVectorLength_successful",
+        "test_getParameterVectorLength_unsuccessful",
+        "test_generateParametricJobs",
     ]
-    
+
 except ImportError:
     # If DIRACCommon tests can't be imported, just run the backward compatibility test
-    __all__ = ['test_backward_compatibility_import']
+    __all__ = ["test_backward_compatibility_import"]

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+"""ClassAd utilities for DIRACCommon"""`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+"""DIRACCommon WorkloadManagementSystem DB utilities"""`