add: coarse retries for agent run

Author: clee-codegen

codegen-examples/examples/swebench_agent_run/eval_cli.py

@@ -294,12 +294,6 @@ def run_eval(
 @click.option("--local", help="Run the evaluation locally.", is_flag=True, default=False)
 @click.option("--push-metrics", help="Push metrics to the database.", is_flag=True, default=False)
 @click.option("--repo", help="The repo to use.", type=str, default=None)
-@click.option(
-    "--num-workers",
-    help="The number of workers to use. This is the number of examples that will be processed concurrently. A large number may lead to rate limiting issues.",
-    type=int,
-    default=5,
-)
 @click.option("--model", help="The model to use.", type=str, default="claude-3-7-sonnet-latest")
 def main(
     use_existing_preds: Optional[str],
@@ -308,7 +302,6 @@ def main(
     instance_id: Optional[str],
     local: bool,
     repo: Optional[str],
-    num_workers: int,
     model: str,
     push_metrics: bool,
 ) -> None:
@@ -324,7 +317,6 @@ def main(
             codebases=None,
             local=local,
             repo=repo,
-            num_workers=num_workers,
             model=model,
         )
     )

codegen-examples/examples/swebench_agent_run/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "click>=8.1.0",
   "codegen",
   "swebench>=3.0.15",
+  "tenacity>=9.0.0",
 ]
 
 [project.optional-dependencies]

codegen-examples/examples/swebench_agent_run/swebench_agent_run/modal_harness/entry_point.py

@@ -13,6 +13,7 @@
 from unittest.mock import patch
 
 import modal as modal_lib
+import tenacity
 from swebench.harness.constants import (
     APPLY_PATCH_FAIL,
     APPLY_PATCH_PASS,
@@ -83,12 +84,38 @@
 )
 
 
+class ShouldRetry(Exception):
+    pass
+
+
 @app.function(timeout=43200)
 async def run_agent_modal(entry: "SweBenchExample", run_id: str, model: str):
     from codegen.extensions.swebench.harness import run_agent_on_entry
 
     """Modal function to process a single example from the SWE-bench dataset."""
-    return run_agent_on_entry(entry, run_id=run_id, model=model)
+    for attempt in tenacity.Retrying(
+        wait=tenacity.wait_exponential_jitter(max=600),
+        retry=tenacity.retry_if_exception_type(ShouldRetry),
+    ):
+        with attempt:
+            try:
+                return run_agent_on_entry(entry, run_id=run_id, model=model)
+            except Exception as e:
+                if any(
+                    msg in str(e).lower()
+                    for msg in (
+                        "rate limit",
+                        "too many requests",
+                        "429",
+                        "throttle",
+                        "quota exceeded",
+                        "capacity",
+                        "limit exceeded",
+                    )
+                ):
+                    raise ShouldRetry() from e
+                else:
+                    raise e
 
 
 @app.function(