Skip to content

Commit 63c9ec4

Browse files
sjarmakclaude
sjarmak
and
claude
committed
feat: restore phase1 curator prompt and launch ground truth batch
Restore the high-performing phase1 curator prompt (af29675) that scored F1=0.749 on ContextBench calibration. The V7/V8 edit-centric prompts scored significantly worse (F1=0.39-0.44). Changes: - Add PHASE1_CLI_PROMPTS + PHASE1_SUFFIX (per-backend, recall-focused) - Add --prompt-version flag: 'phase1' (default) or 'v7' - Phase1 re-enables sg_nls_search (semantic search) alongside keyword - Add SWEAP image repo resolution via Dockerfile # Repo: comments - Fix local_tools variable scope bug in SG token fallback path - Update daytona_curator_runner.py for prompt_version support - Add scripts/run_missing_oracles.sh launcher - Include first 3 ground truth files from test runs Batch run launched: 74 missing SDLC tasks with Opus + hybrid backend. All 221 Org tasks already have oracle files (0 missing). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent f01df6c commit 63c9ec4

File tree

12 files changed

+595
-143
lines changed

12 files changed

+595
-143
lines changed

‎benchmarks/csb_sdlc_fix/ansible-abc-imports-fix-001/tests/ground_truth.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"files": [
3+
"lib/ansible/module_utils/common/_collections_compat.py",
4+
"test/lib/ansible_test/_util/controller/sanity/pylint/plugins/unwanted.py"
5+
],
6+
"symbols": [
7+
{
8+
"file": "lib/ansible/module_utils/common/_collections_compat.py",
9+
"symbol": "_warnings.deprecate",
10+
"repo": null
11+
},
12+
{
13+
"file": "test/lib/ansible_test/_util/controller/sanity/pylint/plugins/unwanted.py",
14+
"symbol": "AnsibleUnwantedChecker",
15+
"repo": null
16+
},
17+
{
18+
"file": "test/lib/ansible_test/_util/controller/sanity/pylint/plugins/unwanted.py",
19+
"symbol": "unwanted_imports",
20+
"repo": null
21+
}
22+
]
23+
}

‎benchmarks/csb_sdlc_fix/ansible-abc-imports-fix-001/tests/ground_truth_meta.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"has_ground_truth": true,
3+
"has_chunk_ground_truth": false,
4+
"ground_truth_source": "curator_agent",
5+
"ground_truth_confidence": "medium",
6+
"task_name": "ansible-abc-imports-fix-001",
7+
"curator_agent_version": "2.0",
8+
"model": "claude-opus-4-6",
9+
"backend": "hybrid",
10+
"timestamp": "2026-03-03T15:04:58.267583+00:00",
11+
"files_count": 2,
12+
"edit_files_count": 0,
13+
"chunks_count": 0,
14+
"symbols_count": 3,
15+
"cost_usd": 3.2322,
16+
"elapsed_sec": 468.1,
17+
"exploration_notes": "The shim file `_collections_compat.py` is a deprecated re-export layer (collections.abc \u2192 named ABCs) with a v2.24 removal deprecation warning. Zero internal ansible-core files still import from it \u2014 the migration is complete. The enforcement gap is in `unwanted.py`: its `unwanted_imports` dict flags `from collections import Mapping` but does NOT flag `from ansible.module_utils.common._collections_compat import Mapping`. To fully align sanity checks with the policy that internal code must use `c"
18+
}

‎benchmarks/csb_sdlc_fix/ansible-module-respawn-fix-001/tests/ground_truth.json‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
{
2+
"files": [
3+
"lib/ansible/module_utils/common/respawn.py",
4+
"lib/ansible/module_utils/_internal/_ansiballz/_respawn.py",
5+
"lib/ansible/module_utils/_internal/_ansiballz/_respawn_wrapper.py",
6+
"lib/ansible/module_utils/_internal/_ansiballz/_loader.py",
7+
"lib/ansible/module_utils/compat/selinux.py",
8+
"lib/ansible/module_utils/basic.py",
9+
"lib/ansible/module_utils/facts/system/selinux.py",
10+
"lib/ansible/modules/dnf.py",
11+
"lib/ansible/modules/dnf5.py",
12+
"lib/ansible/modules/apt.py",
13+
"lib/ansible/modules/apt_repository.py",
14+
"lib/ansible/modules/deb822_repository.py",
15+
"lib/ansible/module_utils/yumdnf.py"
16+
],
17+
"symbols": [
18+
{
19+
"file": "lib/ansible/module_utils/common/respawn.py",
20+
"symbol": "has_respawned",
21+
"repo": null
22+
},
23+
{
24+
"file": "lib/ansible/module_utils/common/respawn.py",
25+
"symbol": "respawn_module",
26+
"repo": null
27+
},
28+
{
29+
"file": "lib/ansible/module_utils/common/respawn.py",
30+
"symbol": "probe_interpreters_for_module",
31+
"repo": null
32+
},
33+
{
34+
"file": "lib/ansible/module_utils/_internal/_ansiballz/_respawn.py",
35+
"symbol": "create_payload",
36+
"repo": null
37+
},
38+
{
39+
"file": "lib/ansible/module_utils/_internal/_ansiballz/_respawn_wrapper.py",
40+
"symbol": "_respawn_main",
41+
"repo": null
42+
},
43+
{
44+
"file": "lib/ansible/module_utils/_internal/_ansiballz/_loader.py",
45+
"symbol": "run_module",
46+
"repo": null
47+
},
48+
{
49+
"file": "lib/ansible/module_utils/compat/selinux.py",
50+
"symbol": "_module_setup",
51+
"repo": null
52+
},
53+
{
54+
"file": "lib/ansible/module_utils/compat/selinux.py",
55+
"symbol": "is_selinux_enabled",
56+
"repo": null
57+
},
58+
{
59+
"file": "lib/ansible/module_utils/compat/selinux.py",
60+
"symbol": "lgetfilecon_raw",
61+
"repo": null
62+
},
63+
{
64+
"file": "lib/ansible/module_utils/compat/selinux.py",
65+
"symbol": "lsetfilecon",
66+
"repo": null
67+
},
68+
{
69+
"file": "lib/ansible/module_utils/basic.py",
70+
"symbol": "HAVE_SELINUX",
71+
"repo": null
72+
},
73+
{
74+
"file": "lib/ansible/modules/dnf.py",
75+
"symbol": "DnfModule._ensure_dnf",
76+
"repo": null
77+
},
78+
{
79+
"file": "lib/ansible/modules/apt.py",
80+
"symbol": "HAS_PYTHON_APT",
81+
"repo": null
82+
},
83+
{
84+
"file": "lib/ansible/module_utils/facts/system/selinux.py",
85+
"symbol": "SelinuxFactCollector",
86+
"repo": null
87+
}
88+
]
89+
}

‎benchmarks/csb_sdlc_fix/ansible-module-respawn-fix-001/tests/ground_truth_agent.json‎

Lines changed: 58 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,135 +1,92 @@
11
{
22
"files": [
3-
{
4-
"repo": "ansible/ansible",
5-
"path": "lib/ansible/module_utils/basic.py"
6-
},
7-
{
8-
"repo": "ansible/ansible",
9-
"path": "lib/ansible/module_utils/compat/selinux.py"
10-
},
11-
{
12-
"repo": "ansible/ansible",
13-
"path": "lib/ansible/module_utils/facts/system/selinux.py"
14-
},
15-
{
16-
"repo": "ansible/ansible",
17-
"path": "lib/ansible/module_utils/yumdnf.py"
18-
},
19-
{
20-
"repo": "ansible/ansible",
21-
"path": "lib/ansible/modules/dnf.py"
22-
},
23-
{
24-
"repo": "ansible/ansible",
25-
"path": "lib/ansible/modules/yum.py"
26-
},
27-
{
28-
"repo": "ansible/ansible",
29-
"path": "lib/ansible/modules/apt.py"
30-
},
31-
{
32-
"repo": "ansible/ansible",
33-
"path": "lib/ansible/modules/apt_repository.py"
34-
},
35-
{
36-
"repo": "ansible/ansible",
37-
"path": "lib/ansible/module_utils/respawn.py"
38-
},
39-
{
40-
"repo": "ansible/ansible",
41-
"path": "test/sanity/ignore.txt"
42-
}
3+
"lib/ansible/module_utils/common/respawn.py",
4+
"lib/ansible/module_utils/_internal/_ansiballz/_respawn.py",
5+
"lib/ansible/module_utils/_internal/_ansiballz/_respawn_wrapper.py",
6+
"lib/ansible/module_utils/_internal/_ansiballz/_loader.py",
7+
"lib/ansible/module_utils/compat/selinux.py",
8+
"lib/ansible/module_utils/basic.py",
9+
"lib/ansible/modules/dnf.py",
10+
"lib/ansible/modules/apt.py",
11+
"lib/ansible/modules/apt_repository.py",
12+
"lib/ansible/modules/dnf5.py",
13+
"lib/ansible/modules/deb822_repository.py"
4314
],
4415
"symbols": [
4516
{
46-
"repo": "ansible/ansible",
47-
"path": "lib/ansible/module_utils/basic.py",
48-
"symbol": "AnsibleModule"
17+
"file": "lib/ansible/module_utils/common/respawn.py",
18+
"symbol": "has_respawned",
19+
"repo": null
4920
},
5021
{
51-
"repo": "ansible/ansible",
52-
"path": "lib/ansible/module_utils/basic.py",
53-
"symbol": "selinux_enabled"
22+
"file": "lib/ansible/module_utils/common/respawn.py",
23+
"symbol": "respawn_module",
24+
"repo": null
5425
},
5526
{
56-
"repo": "ansible/ansible",
57-
"path": "lib/ansible/module_utils/basic.py",
58-
"symbol": "selinux_context"
27+
"file": "lib/ansible/module_utils/common/respawn.py",
28+
"symbol": "probe_interpreters_for_module",
29+
"repo": null
5930
},
6031
{
61-
"repo": "ansible/ansible",
62-
"path": "lib/ansible/module_utils/compat/selinux.py",
63-
"symbol": "selinux_getenforcemode"
32+
"file": "lib/ansible/module_utils/_internal/_ansiballz/_respawn.py",
33+
"symbol": "create_payload",
34+
"repo": null
6435
},
6536
{
66-
"repo": "ansible/ansible",
67-
"path": "lib/ansible/module_utils/compat/selinux.py",
68-
"symbol": "selinux_getpolicytype"
37+
"file": "lib/ansible/module_utils/_internal/_ansiballz/_respawn_wrapper.py",
38+
"symbol": "_respawn_main",
39+
"repo": null
6940
},
7041
{
71-
"repo": "ansible/ansible",
72-
"path": "lib/ansible/module_utils/facts/system/selinux.py",
73-
"symbol": "SelinuxFactCollector"
42+
"file": "lib/ansible/module_utils/_internal/_ansiballz/_loader.py",
43+
"symbol": "run_module",
44+
"repo": null
7445
},
7546
{
76-
"repo": "ansible/ansible",
77-
"path": "lib/ansible/module_utils/yumdnf.py",
78-
"symbol": "YumDnf"
47+
"file": "lib/ansible/module_utils/_internal/_ansiballz/_loader.py",
48+
"symbol": "_run_module",
49+
"repo": null
7950
},
8051
{
81-
"repo": "ansible/ansible",
82-
"path": "lib/ansible/module_utils/yumdnf.py",
83-
"symbol": "yumdnf_argument_spec"
84-
}
85-
],
86-
"chain": [
52+
"file": "lib/ansible/module_utils/compat/selinux.py",
53+
"symbol": "_module_setup",
54+
"repo": null
55+
},
8756
{
88-
"repo": "ansible/ansible",
89-
"path": "lib/ansible/module_utils/respawn.py",
90-
"symbol": "respawn_module"
57+
"file": "lib/ansible/module_utils/compat/selinux.py",
58+
"symbol": "is_selinux_enabled",
59+
"repo": null
9160
},
9261
{
93-
"repo": "ansible/ansible",
94-
"path": "lib/ansible/modules/dnf.py",
95-
"symbol": "main"
62+
"file": "lib/ansible/module_utils/compat/selinux.py",
63+
"symbol": "lgetfilecon_raw",
64+
"repo": null
9665
},
9766
{
98-
"repo": "ansible/ansible",
99-
"path": "lib/ansible/modules/yum.py",
100-
"symbol": "main"
67+
"file": "lib/ansible/module_utils/compat/selinux.py",
68+
"symbol": "matchpathcon",
69+
"repo": null
10170
},
10271
{
103-
"repo": "ansible/ansible",
104-
"path": "lib/ansible/modules/apt.py",
105-
"symbol": "main"
72+
"file": "lib/ansible/module_utils/basic.py",
73+
"symbol": "HAVE_SELINUX",
74+
"repo": null
10675
},
10776
{
108-
"repo": "ansible/ansible",
109-
"path": "lib/ansible/modules/apt_repository.py",
110-
"symbol": "main"
77+
"file": "lib/ansible/module_utils/basic.py",
78+
"symbol": "selinux_enabled",
79+
"repo": null
11180
},
11281
{
113-
"repo": "ansible/ansible",
114-
"path": "lib/ansible/module_utils/compat/selinux.py",
115-
"symbol": "selinux_getenforcemode"
82+
"file": "lib/ansible/module_utils/basic.py",
83+
"symbol": "selinux_context",
84+
"repo": null
11685
},
11786
{
118-
"repo": "ansible/ansible",
119-
"path": "lib/ansible/module_utils/facts/system/selinux.py",
120-
"symbol": "SelinuxFactCollector"
87+
"file": "lib/ansible/module_utils/basic.py",
88+
"symbol": "set_context_if_different",
89+
"repo": null
12190
}
122-
],
123-
"text": "The task requires implementing a module respawn mechanism for ansible modules that need system-specific Python bindings. Key findings:\n\n1. **lib/ansible/module_utils/compat/selinux.py** - Existing SELinux compat shim that uses ctypes to access libselinux.so without the Python bindings. Referenced in test/sanity/ignore.txt with import-skip annotations for Python 3.9/3.10/3.11.\n\n2. **lib/ansible/module_utils/basic.py** - Core AnsibleModule class with selinux_enabled/selinux_context methods that need to be refactored away from libselinux-python dependency.\n\n3. **lib/ansible/module_utils/facts/system/selinux.py** - SELinux fact collector that imports from the compat selinux module.\n\n4. **lib/ansible/module_utils/yumdnf.py** - Shared base class (YumDnf) and argument spec for dnf/yum modules.\n\n5. **lib/ansible/modules/dnf.py**, **yum.py**, **apt.py**, **apt_repository.py** - Package manager modules that need to integrate with the respawn API to re-execute under compatible system interpreters when Python bindings (dnf, rpm, apt) are unavailable.\n\n6. **lib/ansible/module_utils/respawn.py** - New module providing the respawn API allowing modules to detect missing bindings and re-execute under a compatible system Python interpreter.\n\n7. **test/sanity/ignore.txt** - Contains sanity test ignore entries for the selinux compat module, relevant for verifying no regressions.",
124-
"_metadata": {
125-
"generator": "context_retrieval_agent",
126-
"model": "claude-sonnet-4-6",
127-
"backend": "hybrid",
128-
"input_tokens": 161334,
129-
"output_tokens": 4356,
130-
"tool_calls": 42,
131-
"elapsed_sec": 73.9,
132-
"cost_usd": 0.5493,
133-
"timestamp": "2026-03-02T00:57:16.258005+00:00"
134-
}
91+
]
13592
}

‎benchmarks/csb_sdlc_fix/ansible-module-respawn-fix-001/tests/ground_truth_meta.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"has_ground_truth": true,
3+
"has_chunk_ground_truth": false,
4+
"ground_truth_source": "curator_agent",
5+
"ground_truth_confidence": "medium",
6+
"task_name": "ansible-module-respawn-fix-001",
7+
"curator_agent_version": "2.0",
8+
"model": "claude-opus-4-6",
9+
"backend": "local",
10+
"timestamp": "2026-03-03T15:12:14.654624+00:00",
11+
"files_count": 11,
12+
"edit_files_count": 0,
13+
"chunks_count": 0,
14+
"symbols_count": 15,
15+
"cost_usd": 0.8387,
16+
"elapsed_sec": 154.2,
17+
"exploration_notes": "The task involves 3 interconnected subsystems: (1) The module respawn API in `module_utils/common/respawn.py` with its internal AnsiballZ implementation (`_respawn.py`, `_respawn_wrapper.py`, `_loader.py`) \u2014 this provides `has_respawned()`, `respawn_module()`, and `probe_interpreters_for_module()` for re-executing modules under compatible interpreters; (2) The ctypes-based SELinux compatibility layer in `module_utils/compat/selinux.py` which replaces the `libselinux-python` dependency by directl"
18+
}

0 commit comments

Comments
 (0)