Skip to content

Commit 35d5400

Browse files
sjarmakclaude
sjarmak
and
claude
committed
fix: security oracle — add repo prefix to bare-path oracle entries
10 security tasks (ccx-vuln-remed-161 through 170) had bare file paths in oracle_answer.json without repo context (e.g., "pkg/registry/rbac/...") causing _normalize_file_entry to misparse them as repo="pkg/registry". Fixed by adding proper sg-evals mirror repo prefixes to each task's oracle entries and re-hydrating task_spec.json files. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 7fbc48c commit 35d5400

File tree

21 files changed

+586
-165
lines changed

21 files changed

+586
-165
lines changed

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-161/tests/oracle_answer.json‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,33 @@
11
{
22
"files": [
3-
"pkg/registry/rbac/clusterrole/policybased/storage.go",
4-
"pkg/registry/rbac/escalation_check.go",
5-
"pkg/registry/rbac/clusterrolebinding/policybased/storage.go",
6-
"pkg/registry/rbac/validation/rule.go",
7-
"pkg/registry/rbac/clusterrole/registry.go",
8-
"pkg/registry/rbac/clusterrolebinding/registry.go",
9-
"pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go"
3+
{
4+
"repo": "sg-evals/kubernetes--v1.32.0",
5+
"path": "pkg/registry/rbac/clusterrole/policybased/storage.go"
6+
},
7+
{
8+
"repo": "sg-evals/kubernetes--v1.32.0",
9+
"path": "pkg/registry/rbac/escalation_check.go"
10+
},
11+
{
12+
"repo": "sg-evals/kubernetes--v1.32.0",
13+
"path": "pkg/registry/rbac/clusterrolebinding/policybased/storage.go"
14+
},
15+
{
16+
"repo": "sg-evals/kubernetes--v1.32.0",
17+
"path": "pkg/registry/rbac/validation/rule.go"
18+
},
19+
{
20+
"repo": "sg-evals/kubernetes--v1.32.0",
21+
"path": "pkg/registry/rbac/clusterrole/registry.go"
22+
},
23+
{
24+
"repo": "sg-evals/kubernetes--v1.32.0",
25+
"path": "pkg/registry/rbac/clusterrolebinding/registry.go"
26+
},
27+
{
28+
"repo": "sg-evals/kubernetes--v1.32.0",
29+
"path": "pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go"
30+
}
1031
],
1132
"symbols": [],
1233
"repo_set_id": "kubernetes-ecosystem"

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-161/tests/task_spec.json‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,32 +18,32 @@
1818
"oracle": {
1919
"required_files": [
2020
{
21-
"repo": "pkg/registry",
22-
"path": "rbac/clusterrole/policybased/storage.go"
21+
"repo": "sg-evals/kubernetes--v1.32.0",
22+
"path": "pkg/registry/rbac/clusterrole/policybased/storage.go"
2323
},
2424
{
25-
"repo": "pkg/registry",
26-
"path": "rbac/escalation_check.go"
25+
"repo": "sg-evals/kubernetes--v1.32.0",
26+
"path": "pkg/registry/rbac/escalation_check.go"
2727
},
2828
{
29-
"repo": "pkg/registry",
30-
"path": "rbac/clusterrolebinding/policybased/storage.go"
29+
"repo": "sg-evals/kubernetes--v1.32.0",
30+
"path": "pkg/registry/rbac/clusterrolebinding/policybased/storage.go"
3131
},
3232
{
33-
"repo": "pkg/registry",
34-
"path": "rbac/validation/rule.go"
33+
"repo": "sg-evals/kubernetes--v1.32.0",
34+
"path": "pkg/registry/rbac/validation/rule.go"
3535
},
3636
{
37-
"repo": "pkg/registry",
38-
"path": "rbac/clusterrole/registry.go"
37+
"repo": "sg-evals/kubernetes--v1.32.0",
38+
"path": "pkg/registry/rbac/clusterrole/registry.go"
3939
},
4040
{
41-
"repo": "pkg/registry",
42-
"path": "rbac/clusterrolebinding/registry.go"
41+
"repo": "sg-evals/kubernetes--v1.32.0",
42+
"path": "pkg/registry/rbac/clusterrolebinding/registry.go"
4343
},
4444
{
45-
"repo": "pkg/controller",
46-
"path": "clusterroleaggregation/clusterroleaggregation_controller.go"
45+
"repo": "sg-evals/kubernetes--v1.32.0",
46+
"path": "pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go"
4747
}
4848
],
4949
"required_symbols": [],

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-162/tests/oracle_answer.json‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,37 @@
11
{
22
"files": [
3-
"plugin/pkg/admission/serviceaccount/admission.go",
4-
"pkg/controller/serviceaccount/tokengetter.go",
5-
"pkg/registry/core/serviceaccount/storage/storage.go",
6-
"pkg/registry/core/serviceaccount/strategy.go",
7-
"pkg/serviceaccount/claims.go",
8-
"pkg/serviceaccount/legacy.go",
9-
"staging/src/k8s.io/api/core/v1/types.go",
10-
"staging/src/k8s.io/client-go/applyconfigurations/core/v1/serviceaccount.go"
3+
{
4+
"repo": "sg-evals/kubernetes--v1.32.0",
5+
"path": "plugin/pkg/admission/serviceaccount/admission.go"
6+
},
7+
{
8+
"repo": "sg-evals/kubernetes--v1.32.0",
9+
"path": "pkg/controller/serviceaccount/tokengetter.go"
10+
},
11+
{
12+
"repo": "sg-evals/kubernetes--v1.32.0",
13+
"path": "pkg/registry/core/serviceaccount/storage/storage.go"
14+
},
15+
{
16+
"repo": "sg-evals/kubernetes--v1.32.0",
17+
"path": "pkg/registry/core/serviceaccount/strategy.go"
18+
},
19+
{
20+
"repo": "sg-evals/kubernetes--v1.32.0",
21+
"path": "pkg/serviceaccount/claims.go"
22+
},
23+
{
24+
"repo": "sg-evals/kubernetes--v1.32.0",
25+
"path": "pkg/serviceaccount/legacy.go"
26+
},
27+
{
28+
"repo": "sg-evals/kubernetes--v1.32.0",
29+
"path": "staging/src/k8s.io/api/core/v1/types.go"
30+
},
31+
{
32+
"repo": "sg-evals/kubernetes--v1.32.0",
33+
"path": "staging/src/k8s.io/client-go/applyconfigurations/core/v1/serviceaccount.go"
34+
}
1135
],
1236
"symbols": [],
1337
"repo_set_id": "kubernetes-ecosystem"

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-162/tests/task_spec.json‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,36 +18,36 @@
1818
"oracle": {
1919
"required_files": [
2020
{
21-
"repo": "plugin/pkg",
22-
"path": "admission/serviceaccount/admission.go"
21+
"repo": "sg-evals/kubernetes--v1.32.0",
22+
"path": "plugin/pkg/admission/serviceaccount/admission.go"
2323
},
2424
{
25-
"repo": "pkg/controller",
26-
"path": "serviceaccount/tokengetter.go"
25+
"repo": "sg-evals/kubernetes--v1.32.0",
26+
"path": "pkg/controller/serviceaccount/tokengetter.go"
2727
},
2828
{
29-
"repo": "pkg/registry",
30-
"path": "core/serviceaccount/storage/storage.go"
29+
"repo": "sg-evals/kubernetes--v1.32.0",
30+
"path": "pkg/registry/core/serviceaccount/storage/storage.go"
3131
},
3232
{
33-
"repo": "pkg/registry",
34-
"path": "core/serviceaccount/strategy.go"
33+
"repo": "sg-evals/kubernetes--v1.32.0",
34+
"path": "pkg/registry/core/serviceaccount/strategy.go"
3535
},
3636
{
37-
"repo": "pkg/serviceaccount",
38-
"path": "claims.go"
37+
"repo": "sg-evals/kubernetes--v1.32.0",
38+
"path": "pkg/serviceaccount/claims.go"
3939
},
4040
{
41-
"repo": "pkg/serviceaccount",
42-
"path": "legacy.go"
41+
"repo": "sg-evals/kubernetes--v1.32.0",
42+
"path": "pkg/serviceaccount/legacy.go"
4343
},
4444
{
45-
"repo": "staging/src",
46-
"path": "k8s.io/api/core/v1/types.go"
45+
"repo": "sg-evals/kubernetes--v1.32.0",
46+
"path": "staging/src/k8s.io/api/core/v1/types.go"
4747
},
4848
{
49-
"repo": "staging/src",
50-
"path": "k8s.io/client-go/applyconfigurations/core/v1/serviceaccount.go"
49+
"repo": "sg-evals/kubernetes--v1.32.0",
50+
"path": "staging/src/k8s.io/client-go/applyconfigurations/core/v1/serviceaccount.go"
5151
}
5252
],
5353
"required_symbols": [],

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-163/tests/oracle_answer.json‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,25 @@
11
{
22
"files": [
3-
"source/common/http/conn_manager_utility.cc",
4-
"source/common/http/headers.h",
5-
"envoy/http/header_validator.h",
6-
"source/common/http/matching/inputs.h",
7-
"source/common/http/matching/data_impl.h"
3+
{
4+
"repo": "sg-evals/envoy--v1.31.2",
5+
"path": "source/common/http/conn_manager_utility.cc"
6+
},
7+
{
8+
"repo": "sg-evals/envoy--v1.31.2",
9+
"path": "source/common/http/headers.h"
10+
},
11+
{
12+
"repo": "sg-evals/envoy--v1.31.2",
13+
"path": "envoy/http/header_validator.h"
14+
},
15+
{
16+
"repo": "sg-evals/envoy--v1.31.2",
17+
"path": "source/common/http/matching/inputs.h"
18+
},
19+
{
20+
"repo": "sg-evals/envoy--v1.31.2",
21+
"path": "source/common/http/matching/data_impl.h"
22+
}
823
],
924
"symbols": [],
1025
"repo_set_id": "envoy-service-mesh"

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-163/tests/task_spec.json‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,24 @@
1818
"oracle": {
1919
"required_files": [
2020
{
21-
"repo": "source/common",
22-
"path": "http/conn_manager_utility.cc"
21+
"repo": "sg-evals/envoy--v1.31.2",
22+
"path": "source/common/http/conn_manager_utility.cc"
2323
},
2424
{
25-
"repo": "source/common",
26-
"path": "http/headers.h"
25+
"repo": "sg-evals/envoy--v1.31.2",
26+
"path": "source/common/http/headers.h"
2727
},
2828
{
29-
"repo": "envoy/http",
30-
"path": "header_validator.h"
29+
"repo": "sg-evals/envoy--v1.31.2",
30+
"path": "envoy/http/header_validator.h"
3131
},
3232
{
33-
"repo": "source/common",
34-
"path": "http/matching/inputs.h"
33+
"repo": "sg-evals/envoy--v1.31.2",
34+
"path": "source/common/http/matching/inputs.h"
3535
},
3636
{
37-
"repo": "source/common",
38-
"path": "http/matching/data_impl.h"
37+
"repo": "sg-evals/envoy--v1.31.2",
38+
"path": "source/common/http/matching/data_impl.h"
3939
}
4040
],
4141
"required_symbols": [],

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-164/tests/oracle_answer.json‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,33 @@
11
{
22
"files": [
3-
"api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto",
4-
"api/envoy/service/auth/v3/external_auth.proto",
5-
"source/extensions/filters/common/ext_authz/ext_authz.h",
6-
"source/extensions/filters/network/ext_authz/ext_authz.h",
7-
"source/extensions/filters/network/ext_authz/config.h",
8-
"api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto",
9-
"api/envoy/service/auth/v2/external_auth.proto"
3+
{
4+
"repo": "sg-evals/envoy--v1.31.2",
5+
"path": "api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto"
6+
},
7+
{
8+
"repo": "sg-evals/envoy--v1.31.2",
9+
"path": "api/envoy/service/auth/v3/external_auth.proto"
10+
},
11+
{
12+
"repo": "sg-evals/envoy--v1.31.2",
13+
"path": "source/extensions/filters/common/ext_authz/ext_authz.h"
14+
},
15+
{
16+
"repo": "sg-evals/envoy--v1.31.2",
17+
"path": "source/extensions/filters/network/ext_authz/ext_authz.h"
18+
},
19+
{
20+
"repo": "sg-evals/envoy--v1.31.2",
21+
"path": "source/extensions/filters/network/ext_authz/config.h"
22+
},
23+
{
24+
"repo": "sg-evals/envoy--v1.31.2",
25+
"path": "api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto"
26+
},
27+
{
28+
"repo": "sg-evals/envoy--v1.31.2",
29+
"path": "api/envoy/service/auth/v2/external_auth.proto"
30+
}
1031
],
1132
"symbols": [],
1233
"repo_set_id": "envoy-service-mesh"

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-164/tests/task_spec.json‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,32 +18,32 @@
1818
"oracle": {
1919
"required_files": [
2020
{
21-
"repo": "api/envoy",
22-
"path": "extensions/filters/http/ext_authz/v3/ext_authz.proto"
21+
"repo": "sg-evals/envoy--v1.31.2",
22+
"path": "api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto"
2323
},
2424
{
25-
"repo": "api/envoy",
26-
"path": "service/auth/v3/external_auth.proto"
25+
"repo": "sg-evals/envoy--v1.31.2",
26+
"path": "api/envoy/service/auth/v3/external_auth.proto"
2727
},
2828
{
29-
"repo": "source/extensions",
30-
"path": "filters/common/ext_authz/ext_authz.h"
29+
"repo": "sg-evals/envoy--v1.31.2",
30+
"path": "source/extensions/filters/common/ext_authz/ext_authz.h"
3131
},
3232
{
33-
"repo": "source/extensions",
34-
"path": "filters/network/ext_authz/ext_authz.h"
33+
"repo": "sg-evals/envoy--v1.31.2",
34+
"path": "source/extensions/filters/network/ext_authz/ext_authz.h"
3535
},
3636
{
37-
"repo": "source/extensions",
38-
"path": "filters/network/ext_authz/config.h"
37+
"repo": "sg-evals/envoy--v1.31.2",
38+
"path": "source/extensions/filters/network/ext_authz/config.h"
3939
},
4040
{
41-
"repo": "api/envoy",
42-
"path": "config/filter/http/ext_authz/v2/ext_authz.proto"
41+
"repo": "sg-evals/envoy--v1.31.2",
42+
"path": "api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto"
4343
},
4444
{
45-
"repo": "api/envoy",
46-
"path": "service/auth/v2/external_auth.proto"
45+
"repo": "sg-evals/envoy--v1.31.2",
46+
"path": "api/envoy/service/auth/v2/external_auth.proto"
4747
}
4848
],
4949
"required_symbols": [],

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-165/tests/oracle_answer.json‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,21 @@
11
{
22
"files": [
3-
"django/db/backends/utils.py",
4-
"django/db/models/sql/compiler.py",
5-
"django/db/models/query.py",
6-
"django/db/backends/mysql/base.py"
3+
{
4+
"repo": "sg-evals/django--5.1",
5+
"path": "django/db/backends/utils.py"
6+
},
7+
{
8+
"repo": "sg-evals/django--5.1",
9+
"path": "django/db/models/sql/compiler.py"
10+
},
11+
{
12+
"repo": "sg-evals/django--5.1",
13+
"path": "django/db/models/query.py"
14+
},
15+
{
16+
"repo": "sg-evals/django--5.1",
17+
"path": "django/db/backends/mysql/base.py"
18+
}
719
],
820
"symbols": [],
921
"repo_set_id": "django-web-framework"

‎benchmarks/ccb_mcp_security/ccx-vuln-remed-165/tests/task_spec.json‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,20 @@
1818
"oracle": {
1919
"required_files": [
2020
{
21-
"repo": "django/db",
22-
"path": "backends/utils.py"
21+
"repo": "sg-evals/django--5.1",
22+
"path": "django/db/backends/utils.py"
2323
},
2424
{
25-
"repo": "django/db",
26-
"path": "models/sql/compiler.py"
25+
"repo": "sg-evals/django--5.1",
26+
"path": "django/db/models/sql/compiler.py"
2727
},
2828
{
29-
"repo": "django/db",
30-
"path": "models/query.py"
29+
"repo": "sg-evals/django--5.1",
30+
"path": "django/db/models/query.py"
3131
},
3232
{
33-
"repo": "django/db",
34-
"path": "backends/mysql/base.py"
33+
"repo": "sg-evals/django--5.1",
34+
"path": "django/db/backends/mysql/base.py"
3535
}
3636
],
3737
"required_symbols": [],

0 commit comments

Comments
 (0)