|
279 | 279 | } |
280 | 280 | ] |
281 | 281 | }, |
282 | | - "swebenchpro": { |
283 | | - "_note": "36 repo+commit combinations for 36 SWE-bench Pro tasks. Verified via Sourcegraph GraphQL API 2026-02-04: 32/36 indexed (26 newly pushed + 6 existing), 4 missing (protonmail/webclients commits not found in source repo).", |
284 | | - "_verified": "2026-02-04T17:45:00Z", |
285 | | - "_status": "32_OF_36_INDEXED", |
286 | | - "_indexed_count": 32, |
287 | | - "_missing_count": 4, |
288 | | - "repos": [ |
289 | | - { |
290 | | - "sg_name": "sg-evals/ansible--379058e1", |
291 | | - "github_repo": "ansible/ansible", |
292 | | - "commit": "379058e10f3dbc0fdcaf80394bd09b18927e7d33", |
293 | | - "_indexed": true |
294 | | - }, |
295 | | - { |
296 | | - "sg_name": "sg-evals/ansible--4c5ce5a1", |
297 | | - "github_repo": "ansible/ansible", |
298 | | - "commit": "4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6", |
299 | | - "_indexed": true |
300 | | - }, |
301 | | - { |
302 | | - "sg_name": "sg-evals/ansible--811093f0", |
303 | | - "github_repo": "ansible/ansible", |
304 | | - "commit": "811093f0225caa4dd33890933150a81c6a6d5226", |
305 | | - "_indexed": true |
306 | | - }, |
307 | | - { |
308 | | - "sg_name": "sg-evals/ansible--b2a289dc", |
309 | | - "github_repo": "ansible/ansible", |
310 | | - "commit": "b2a289dcbb702003377221e25f62c8a3608f0e89", |
311 | | - "_indexed": true |
312 | | - }, |
313 | | - { |
314 | | - "sg_name": "sg-evals/ansible--e40889e7", |
315 | | - "github_repo": "ansible/ansible", |
316 | | - "commit": "e40889e7112ae00a21a2c74312b330e67a766cc0", |
317 | | - "_indexed": true |
318 | | - }, |
319 | | - { |
320 | | - "sg_name": "sg-evals/element-web--cf3c899d", |
321 | | - "github_repo": "element-hq/element-web", |
322 | | - "commit": "cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85", |
323 | | - "_indexed": true |
324 | | - }, |
325 | | - { |
326 | | - "sg_name": "sg-evals/element-web--f14374a5", |
327 | | - "github_repo": "element-hq/element-web", |
328 | | - "commit": "f14374a51c153f64f313243f2df6ea4971db4e15", |
329 | | - "_indexed": true |
330 | | - }, |
331 | | - { |
332 | | - "sg_name": "sg-evals/flipt--3d5a345f", |
333 | | - "github_repo": "flipt-io/flipt", |
334 | | - "commit": "3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8", |
335 | | - "_indexed": true |
336 | | - }, |
337 | | - { |
338 | | - "sg_name": "sg-evals/flipt--9f8127f2", |
339 | | - "github_repo": "flipt-io/flipt", |
340 | | - "commit": "9f8127f225a86245fa35dca4885c2daef824ee55", |
341 | | - "_indexed": true |
342 | | - }, |
343 | | - { |
344 | | - "sg_name": "sg-evals/flipt--b433bd05", |
345 | | - "github_repo": "flipt-io/flipt", |
346 | | - "commit": "b433bd05ce405837804693bebd5f4b88d87133c8", |
347 | | - "_indexed": true |
348 | | - }, |
349 | | - { |
350 | | - "sg_name": "sg-evals/flipt--c188284f", |
351 | | - "github_repo": "flipt-io/flipt", |
352 | | - "commit": "c188284ff0c094a4ee281afebebd849555ebee59", |
353 | | - "_indexed": true |
354 | | - }, |
355 | | - { |
356 | | - "sg_name": "sg-evals/navidrome--9c3b4561", |
357 | | - "github_repo": "navidrome/navidrome", |
358 | | - "commit": "9c3b4561652a15846993d477003e111f0df0c585", |
359 | | - "_indexed": true |
360 | | - }, |
361 | | - { |
362 | | - "sg_name": "sg-evals/navidrome--d0dceae0", |
363 | | - "github_repo": "navidrome/navidrome", |
364 | | - "commit": "d0dceae0943b8df16e579c2d9437e11760a0626a", |
365 | | - "_indexed": true |
366 | | - }, |
367 | | - { |
368 | | - "sg_name": "sg-evals/nodebb--76c6e302", |
369 | | - "github_repo": "nodebb/nodebb", |
370 | | - "commit": "76c6e30282906ac664f2c9278fc90999b27b1f48", |
371 | | - "_indexed": true |
372 | | - }, |
373 | | - { |
374 | | - "sg_name": "sg-evals/nodebb--eb49a649", |
375 | | - "github_repo": "nodebb/nodebb", |
376 | | - "commit": "eb49a64974ca844bca061744fb3383f5d13b02ad", |
377 | | - "_indexed": true |
378 | | - }, |
379 | | - { |
380 | | - "sg_name": "sg-evals/nodebb--f1a80d48", |
381 | | - "github_repo": "nodebb/nodebb", |
382 | | - "commit": "f1a80d48cc45877fcbadf34c2345dd9709722c7f", |
383 | | - "_indexed": true |
384 | | - }, |
385 | | - { |
386 | | - "sg_name": "sg-evals/openlibrary--7f6b722a", |
387 | | - "github_repo": "internetarchive/openlibrary", |
388 | | - "commit": "7f6b722a10f822171501d027cad60afe53337732", |
389 | | - "_indexed": true |
390 | | - }, |
391 | | - { |
392 | | - "sg_name": "sg-evals/openlibrary--92db3454", |
393 | | - "github_repo": "internetarchive/openlibrary", |
394 | | - "commit": "92db3454aeaa02f89b4cdbc3103f7e95c9759f92", |
395 | | - "_indexed": true |
396 | | - }, |
397 | | - { |
398 | | - "sg_name": "sg-evals/openlibrary--c506c1b0", |
399 | | - "github_repo": "internetarchive/openlibrary", |
400 | | - "commit": "c506c1b0b678892af5cb22c1c1dbc35d96787a0a", |
401 | | - "_indexed": true |
402 | | - }, |
403 | | - { |
404 | | - "sg_name": "sg-evals/openlibrary--d109cc7e", |
405 | | - "github_repo": "internetarchive/openlibrary", |
406 | | - "commit": "d109cc7e6e161170391f98f9a6fa1d02534c18e4", |
407 | | - "_indexed": true |
408 | | - }, |
409 | | - { |
410 | | - "sg_name": "sg-evals/qutebrowser--233cb1cc", |
411 | | - "github_repo": "qutebrowser/qutebrowser", |
412 | | - "commit": "233cb1cc48635130e5602549856a6fa4ab4c087f", |
413 | | - "_indexed": true |
414 | | - }, |
415 | | - { |
416 | | - "sg_name": "sg-evals/qutebrowser--394bfaed", |
417 | | - "github_repo": "qutebrowser/qutebrowser", |
418 | | - "commit": "394bfaed6544c952c6b3463751abab3176ad4997", |
419 | | - "_indexed": true |
420 | | - }, |
421 | | - { |
422 | | - "sg_name": "sg-evals/qutebrowser--3fd8e129", |
423 | | - "github_repo": "qutebrowser/qutebrowser", |
424 | | - "commit": "3fd8e12949b8feda401930574facf09dd4180bba", |
425 | | - "_indexed": true |
426 | | - }, |
427 | | - { |
428 | | - "sg_name": "sg-evals/qutebrowser--e5340c44", |
429 | | - "github_repo": "qutebrowser/qutebrowser", |
430 | | - "commit": "e5340c449f23608803c286da0563b62f58ba25b0", |
431 | | - "_indexed": true |
432 | | - }, |
433 | | - { |
434 | | - "sg_name": "sg-evals/teleport--0415e422", |
435 | | - "github_repo": "gravitational/teleport", |
436 | | - "commit": "0415e422f12454db0c22316cf3eaa5088d6b6322", |
437 | | - "_indexed": true |
438 | | - }, |
439 | | - { |
440 | | - "sg_name": "sg-evals/teleport--3587cca7", |
441 | | - "github_repo": "gravitational/teleport", |
442 | | - "commit": "3587cca7840f636489449113969a5066025dd5bf", |
443 | | - "_indexed": true |
444 | | - }, |
445 | | - { |
446 | | - "sg_name": "sg-evals/teleport--7744f72c", |
447 | | - "github_repo": "gravitational/teleport", |
448 | | - "commit": "7744f72c6eb631791434b648ba41083b5f6d2278", |
449 | | - "_indexed": true |
450 | | - }, |
451 | | - { |
452 | | - "sg_name": "sg-evals/teleport--8302d467", |
453 | | - "github_repo": "gravitational/teleport", |
454 | | - "commit": "8302d467d160f869b77184e262adbe2fbc95d9ba", |
455 | | - "_indexed": true |
456 | | - }, |
457 | | - { |
458 | | - "sg_name": "sg-evals/tutanota--f373ac38", |
459 | | - "github_repo": "tutao/tutanota", |
460 | | - "commit": "f373ac3808deefce8183dad8d16729839cc330c1", |
461 | | - "_indexed": true |
462 | | - }, |
463 | | - { |
464 | | - "sg_name": "sg-evals/vuls--139f3a81", |
465 | | - "github_repo": "future-architect/vuls", |
466 | | - "commit": "139f3a81b66c47e6d8f70ce6c4afe7a9196a6ea8", |
467 | | - "_indexed": true |
468 | | - }, |
469 | | - { |
470 | | - "sg_name": "sg-evals/vuls--4c04acbd", |
471 | | - "github_repo": "future-architect/vuls", |
472 | | - "commit": "4c04acbd9ea5b073efe999e33381fa9f399d6f27", |
473 | | - "_indexed": true |
474 | | - }, |
475 | | - { |
476 | | - "sg_name": "sg-evals/vuls--d18e7a75", |
477 | | - "github_repo": "future-architect/vuls", |
478 | | - "commit": "d18e7a751d07260d75ce3ba0cd67c4a6aebfd967", |
479 | | - "_indexed": true |
480 | | - }, |
481 | | - { |
482 | | - "sg_name": "sg-evals/webclients--369fd37d", |
483 | | - "github_repo": "protonmail/webclients", |
484 | | - "commit": "369fd37de29c14c690cb3b1c09a949189734026f", |
485 | | - "_indexed": false, |
486 | | - "_note": "Commit not found in source repo" |
487 | | - }, |
488 | | - { |
489 | | - "sg_name": "sg-evals/webclients--8be4f6cb", |
490 | | - "github_repo": "protonmail/webclients", |
491 | | - "commit": "8be4f6cb9380fcd2e67bcb18cef931ae0d4b869c", |
492 | | - "_indexed": false, |
493 | | - "_note": "Commit not found in source repo" |
494 | | - }, |
495 | | - { |
496 | | - "sg_name": "sg-evals/webclients--c6f65d20", |
497 | | - "github_repo": "protonmail/webclients", |
498 | | - "commit": "c6f65d205c401350a226bb005f42fac1754b0b5b", |
499 | | - "_indexed": false, |
500 | | - "_note": "Commit not found in source repo" |
501 | | - }, |
502 | | - { |
503 | | - "sg_name": "sg-evals/webclients--caf10ba9", |
504 | | - "github_repo": "protonmail/webclients", |
505 | | - "commit": "caf10ba9ab2677761c88522d1ba8ad025779c492", |
506 | | - "_indexed": false, |
507 | | - "_note": "Commit not found in source repo" |
508 | | - } |
509 | | - ] |
510 | | - }, |
| 282 | + "swebenchpro": [ |
| 283 | + "_note", |
| 284 | + "_verified", |
| 285 | + "_status", |
| 286 | + "_indexed_count", |
| 287 | + "_missing_count", |
| 288 | + "repos" |
| 289 | + ], |
511 | 290 | "tac": { |
512 | 291 | "_note": "Commits extracted from TAC GitLab (the-agent-company.com:8929) on 2026-02-05. copilot-arena-server not present in TAC GitLab backup \u2014 uses --latest fallback.", |
513 | 292 | "_status": "3_of_4_pinned", |
|
0 commit comments