From 570aa1a79e852f9fbb3d42a29ab75beddada5ac4 Mon Sep 17 00:00:00 2001 From: Bence Gadanyi Date: Tue, 27 Jan 2026 10:43:34 +0000 Subject: [PATCH 1/4] chore: disable query reformulation --- .../app/services/ai_processor.py | 7 ++-- .../tests/test_ai_processor.py | 41 +++++++++++-------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/packages/slackBotFunction/app/services/ai_processor.py b/packages/slackBotFunction/app/services/ai_processor.py index 857a9e24e..dc241a52e 100644 --- a/packages/slackBotFunction/app/services/ai_processor.py +++ b/packages/slackBotFunction/app/services/ai_processor.py @@ -6,7 +6,8 @@ """ from app.services.bedrock import query_bedrock -from app.services.query_reformulator import reformulate_query + +# from app.services.query_reformulator import reformulate_query from app.core.config import get_logger from app.core.types import AIProcessorResponse @@ -16,10 +17,10 @@ def process_ai_query(user_query: str, session_id: str | None = None) -> AIProcessorResponse: """shared AI processing logic for both slack and direct invocation""" # reformulate: improves vector search quality in knowledge base - reformulated_query = reformulate_query(user_query) + # reformulated_query = reformulate_query(user_query) # session_id enables conversation continuity across multiple queries - kb_response = query_bedrock(reformulated_query, session_id) + kb_response = query_bedrock(user_query, session_id) logger.info( "response from bedrock", diff --git a/packages/slackBotFunction/tests/test_ai_processor.py b/packages/slackBotFunction/tests/test_ai_processor.py index 8cb9f7fbe..9ca130786 100644 --- a/packages/slackBotFunction/tests/test_ai_processor.py +++ b/packages/slackBotFunction/tests/test_ai_processor.py @@ -11,7 +11,7 @@ class TestAIProcessor: @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_without_session(self, mock_reformulate, mock_bedrock): """new conversation: no session context passed to bedrock""" - mock_reformulate.return_value = "reformulated: How to authenticate EPS API?" + # mock_reformulate.return_value = "reformulated: How to authenticate EPS API?" mock_bedrock.return_value = { "output": {"text": "To authenticate with EPS API, you need..."}, "sessionId": "new-session-abc123", @@ -26,14 +26,14 @@ def test_process_ai_query_without_session(self, mock_reformulate, mock_bedrock): assert result["citations"][0]["title"] == "EPS Authentication Guide" assert "kb_response" in result - mock_reformulate.assert_called_once_with("How to authenticate EPS API?") - mock_bedrock.assert_called_once_with("reformulated: How to authenticate EPS API?", None) + # mock_reformulate.assert_called_once_with("How to authenticate EPS API?") + mock_bedrock.assert_called_once_with("How to authenticate EPS API?", None) @patch("app.services.ai_processor.query_bedrock") @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_with_session(self, mock_reformulate, mock_bedrock): """conversation continuity: existing session maintained across queries""" - mock_reformulate.return_value = "reformulated: What about rate limits?" + # mock_reformulate.return_value = "reformulated: What about rate limits?" mock_bedrock.return_value = { "output": {"text": "EPS API has rate limits of..."}, "sessionId": "existing-session-456", @@ -47,39 +47,42 @@ def test_process_ai_query_with_session(self, mock_reformulate, mock_bedrock): assert result["citations"] == [] assert "kb_response" in result - mock_reformulate.assert_called_once_with("What about rate limits?") - mock_bedrock.assert_called_once_with("reformulated: What about rate limits?", "existing-session-456") + # mock_reformulate.assert_called_once_with("What about rate limits?") + mock_bedrock.assert_called_once_with("What about rate limits?", "existing-session-456") @patch("app.services.ai_processor.query_bedrock") @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_reformulate_error(self, mock_reformulate, mock_bedrock): """graceful degradation: reformulation failure bubbles up""" - mock_reformulate.side_effect = Exception("Query reformulation failed") + # Test disabled as reformulation is currently bypassed + pass + # mock_reformulate.side_effect = Exception("Query reformulation failed") - with pytest.raises(Exception) as exc_info: - process_ai_query("How to authenticate EPS API?") + # with pytest.raises(Exception) as exc_info: + # process_ai_query("How to authenticate EPS API?") - assert "Query reformulation failed" in str(exc_info.value) - mock_bedrock.assert_not_called() + # assert "Query reformulation failed" in str(exc_info.value) + # mock_bedrock.assert_not_called() @patch("app.services.ai_processor.query_bedrock") @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_bedrock_error(self, mock_reformulate, mock_bedrock): """bedrock service failure: error propagated to caller""" - mock_reformulate.return_value = "reformulated query" + # mock_reformulate.return_value = "reformulated query" mock_bedrock.side_effect = Exception("Bedrock service error") with pytest.raises(Exception) as exc_info: process_ai_query("How to authenticate EPS API?") assert "Bedrock service error" in str(exc_info.value) - mock_reformulate.assert_called_once() + # mock_reformulate.assert_called_once() @patch("app.services.ai_processor.query_bedrock") @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_missing_citations(self, mock_reformulate, mock_bedrock): """bedrock response incomplete: citations default to empty list""" - mock_reformulate.return_value = "reformulated query" + """bedrock response incomplete: citations default to empty list""" + # mock_reformulate.return_value = "reformulated query" mock_bedrock.return_value = { "output": {"text": "Response without citations"}, "sessionId": "session-123", @@ -96,7 +99,8 @@ def test_process_ai_query_missing_citations(self, mock_reformulate, mock_bedrock @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_missing_session_id(self, mock_reformulate, mock_bedrock): """bedrock response incomplete: session_id properly handles None""" - mock_reformulate.return_value = "reformulated query" + """bedrock response incomplete: session_id properly handles None""" + # mock_reformulate.return_value = "reformulated query" mock_bedrock.return_value = { "output": {"text": "Response without session"}, "citations": [], @@ -113,7 +117,7 @@ def test_process_ai_query_missing_session_id(self, mock_reformulate, mock_bedroc @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_empty_query(self, mock_reformulate, mock_bedrock): """edge case: empty query still processed through full pipeline""" - mock_reformulate.return_value = "" + # mock_reformulate.return_value = "" mock_bedrock.return_value = { "output": {"text": "Please provide a question"}, "sessionId": "session-empty", @@ -123,14 +127,15 @@ def test_process_ai_query_empty_query(self, mock_reformulate, mock_bedrock): result = process_ai_query("") assert result["text"] == "Please provide a question" - mock_reformulate.assert_called_once_with("") + # mock_reformulate.assert_called_once_with("") mock_bedrock.assert_called_once_with("", None) @patch("app.services.ai_processor.query_bedrock") @patch("app.services.ai_processor.reformulate_query") def test_process_ai_query_includes_raw_response(self, mock_reformulate, mock_bedrock): """slack needs raw bedrock data: kb_response preserved for session handling""" - mock_reformulate.return_value = "reformulated query" + """slack needs raw bedrock data: kb_response preserved for session handling""" + # mock_reformulate.return_value = "reformulated query" raw_response = { "output": {"text": "Test response"}, "sessionId": "test-123", From da1824c2a942bcf1d12155e6c48c28741b7d132a Mon Sep 17 00:00:00 2001 From: Bence Gadanyi Date: Tue, 27 Jan 2026 10:52:35 +0000 Subject: [PATCH 2/4] chore: disable query reformulation --- .../tests/test_ai_processor.py | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/packages/slackBotFunction/tests/test_ai_processor.py b/packages/slackBotFunction/tests/test_ai_processor.py index 9ca130786..68947656b 100644 --- a/packages/slackBotFunction/tests/test_ai_processor.py +++ b/packages/slackBotFunction/tests/test_ai_processor.py @@ -8,8 +8,7 @@ class TestAIProcessor: @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_without_session(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_without_session(self, mock_bedrock): """new conversation: no session context passed to bedrock""" # mock_reformulate.return_value = "reformulated: How to authenticate EPS API?" mock_bedrock.return_value = { @@ -30,8 +29,7 @@ def test_process_ai_query_without_session(self, mock_reformulate, mock_bedrock): mock_bedrock.assert_called_once_with("How to authenticate EPS API?", None) @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_with_session(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_with_session(self, mock_bedrock): """conversation continuity: existing session maintained across queries""" # mock_reformulate.return_value = "reformulated: What about rate limits?" mock_bedrock.return_value = { @@ -51,8 +49,7 @@ def test_process_ai_query_with_session(self, mock_reformulate, mock_bedrock): mock_bedrock.assert_called_once_with("What about rate limits?", "existing-session-456") @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_reformulate_error(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_reformulate_error(self, mock_bedrock): """graceful degradation: reformulation failure bubbles up""" # Test disabled as reformulation is currently bypassed pass @@ -65,8 +62,7 @@ def test_process_ai_query_reformulate_error(self, mock_reformulate, mock_bedrock # mock_bedrock.assert_not_called() @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_bedrock_error(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_bedrock_error(self, mock_bedrock): """bedrock service failure: error propagated to caller""" # mock_reformulate.return_value = "reformulated query" mock_bedrock.side_effect = Exception("Bedrock service error") @@ -78,8 +74,7 @@ def test_process_ai_query_bedrock_error(self, mock_reformulate, mock_bedrock): # mock_reformulate.assert_called_once() @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_missing_citations(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_missing_citations(self, mock_bedrock): """bedrock response incomplete: citations default to empty list""" """bedrock response incomplete: citations default to empty list""" # mock_reformulate.return_value = "reformulated query" @@ -96,8 +91,7 @@ def test_process_ai_query_missing_citations(self, mock_reformulate, mock_bedrock assert result["citations"] == [] # safe default when bedrock omits citations @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_missing_session_id(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_missing_session_id(self, mock_bedrock): """bedrock response incomplete: session_id properly handles None""" """bedrock response incomplete: session_id properly handles None""" # mock_reformulate.return_value = "reformulated query" @@ -114,8 +108,7 @@ def test_process_ai_query_missing_session_id(self, mock_reformulate, mock_bedroc assert result["citations"] == [] @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_empty_query(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_empty_query(self, mock_bedrock): """edge case: empty query still processed through full pipeline""" # mock_reformulate.return_value = "" mock_bedrock.return_value = { @@ -131,8 +124,7 @@ def test_process_ai_query_empty_query(self, mock_reformulate, mock_bedrock): mock_bedrock.assert_called_once_with("", None) @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_includes_raw_response(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_includes_raw_response(self, mock_bedrock): """slack needs raw bedrock data: kb_response preserved for session handling""" """slack needs raw bedrock data: kb_response preserved for session handling""" # mock_reformulate.return_value = "reformulated query" From 0cb4fce7f72b1cdda1a52780b2537c87b009bf86 Mon Sep 17 00:00:00 2001 From: Bence Gadanyi Date: Tue, 27 Jan 2026 11:27:46 +0000 Subject: [PATCH 3/4] chore: change retrieval to 3 maximum results --- packages/slackBotFunction/app/services/bedrock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/slackBotFunction/app/services/bedrock.py b/packages/slackBotFunction/app/services/bedrock.py index 44d020196..a2c4edf89 100644 --- a/packages/slackBotFunction/app/services/bedrock.py +++ b/packages/slackBotFunction/app/services/bedrock.py @@ -44,7 +44,7 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat "knowledgeBaseId": config.KNOWLEDGEBASE_ID, "modelArn": prompt_template.get("model_id", config.RAG_MODEL_ID), "retrievalConfiguration": { - "vectorSearchConfiguration": {"numberOfResults": 5, "overrideSearchType": "SEMANTIC"} + "vectorSearchConfiguration": {"numberOfResults": 3, "overrideSearchType": "SEMANTIC"} }, "generationConfiguration": { "guardrailConfiguration": { From e770347f5997a9180d4f3b06768607896789c10e Mon Sep 17 00:00:00 2001 From: Bence Gadanyi Date: Tue, 27 Jan 2026 12:26:39 +0000 Subject: [PATCH 4/4] chore: automated test pr forwarding --- .../app/slack/slack_handlers.py | 24 ++++++++++++------- .../app/utils/handler_utils.py | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/slackBotFunction/app/slack/slack_handlers.py b/packages/slackBotFunction/app/slack/slack_handlers.py index 734ac2b67..15118453c 100644 --- a/packages/slackBotFunction/app/slack/slack_handlers.py +++ b/packages/slackBotFunction/app/slack/slack_handlers.py @@ -160,14 +160,22 @@ def command_handler(body: Dict[str, Any], command: Dict[str, Any], client: WebCl session_pull_request_id = extract_test_command_params(command.get("text")).get("pr") if session_pull_request_id: logger.info(f"Command in pull request session {session_pull_request_id} from user {user_id}") - forward_to_pull_request_lambda( - body=body, - event={**command, "channel": command.get("channel_id")}, - pull_request_id=session_pull_request_id, - event_id=f"/command-{time.time()}", - store_pull_request_id=False, - type="command", - ) + try: + forward_to_pull_request_lambda( + body=body, + event={**command, "channel": command.get("channel_id")}, + pull_request_id=session_pull_request_id, + event_id=f"/command-{time.time()}", + store_pull_request_id=False, + type="command", + ) + except Exception as e: + logger.error(f"Failed to forward to PR lambda: {e}", extra={"error": traceback.format_exc()}) + client.chat_postEphemeral( + channel=command.get("channel_id"), + user=user_id, + text=f"Failed to forward to PR {session_pull_request_id}. Does the stack exist?", + ) return try: diff --git a/packages/slackBotFunction/app/utils/handler_utils.py b/packages/slackBotFunction/app/utils/handler_utils.py index 44ebc0b1b..bbeaf0b3f 100644 --- a/packages/slackBotFunction/app/utils/handler_utils.py +++ b/packages/slackBotFunction/app/utils/handler_utils.py @@ -184,7 +184,7 @@ def extract_test_command_params(text: str) -> Dict[str, str]: pr_pattern = rf"{prefix}\s*(\d+)\b" q_pattern = r"\bq-?(\d+)(?:-(\d+))?" - pr_match = re.match(pr_pattern, text, flags=re.IGNORECASE) + pr_match = re.search(pr_pattern, text, flags=re.IGNORECASE) if pr_match: params["pr"] = pr_match.group(1)