From b70ed71b4afac0cda81c3cfc9c12e2c890acf4d3 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Thu, 12 Feb 2026 12:10:33 +0000 Subject: [PATCH 01/21] feat: Use hybrid searching --- packages/slackBotFunction/app/services/bedrock.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/slackBotFunction/app/services/bedrock.py b/packages/slackBotFunction/app/services/bedrock.py index 44d020196..df86c3acd 100644 --- a/packages/slackBotFunction/app/services/bedrock.py +++ b/packages/slackBotFunction/app/services/bedrock.py @@ -43,9 +43,7 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat "knowledgeBaseConfiguration": { "knowledgeBaseId": config.KNOWLEDGEBASE_ID, "modelArn": prompt_template.get("model_id", config.RAG_MODEL_ID), - "retrievalConfiguration": { - "vectorSearchConfiguration": {"numberOfResults": 5, "overrideSearchType": "SEMANTIC"} - }, + "retrievalConfiguration": {"vectorSearchConfiguration": {"numberOfResults": 5}}, "generationConfiguration": { "guardrailConfiguration": { "guardrailId": config.GUARD_RAIL_ID, From 9fbdede8bd9958221de52e6c86cf53adf0062fff Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Thu, 12 Feb 2026 13:50:47 +0000 Subject: [PATCH 02/21] feat: Use Semantic Chunking --- packages/cdk/resources/VectorKnowledgeBaseResources.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cdk/resources/VectorKnowledgeBaseResources.ts b/packages/cdk/resources/VectorKnowledgeBaseResources.ts index 37717fd51..f83e5a731 100644 --- a/packages/cdk/resources/VectorKnowledgeBaseResources.ts +++ b/packages/cdk/resources/VectorKnowledgeBaseResources.ts @@ -157,7 +157,7 @@ export class VectorKnowledgeBaseResources extends Construct { // prefix pointed to processed/ to only ingest converted markdown documents const chunkingConfiguration = { - ...ChunkingStrategy.HIERARCHICAL_TITAN.configuration, + ...ChunkingStrategy.SEMANTIC.configuration, hierarchicalChunkingConfiguration: { overlapTokens: 60, levelConfigurations: [ From 82de6d6012f463c9f2e12bfb8ce36b4d1c53a6d1 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Thu, 12 Feb 2026 17:02:18 +0000 Subject: [PATCH 03/21] feat: update inference and chunk configs --- packages/cdk/resources/BedrockPromptSettings.ts | 2 +- .../cdk/resources/VectorKnowledgeBaseResources.ts | 14 ++++++-------- packages/slackBotFunction/app/services/bedrock.py | 2 +- .../slackBotFunction/app/services/prompt_loader.py | 2 +- .../tests/test_bedrock_integration.py | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/cdk/resources/BedrockPromptSettings.ts b/packages/cdk/resources/BedrockPromptSettings.ts index 9526e6e09..3cd518f4d 100644 --- a/packages/cdk/resources/BedrockPromptSettings.ts +++ b/packages/cdk/resources/BedrockPromptSettings.ts @@ -34,7 +34,7 @@ export class BedrockPromptSettings extends Construct { this.inferenceConfig = { temperature: 0, - topP: 0.3, + topP: 0.1, maxTokens: 1024, stopSequences: [ "Human:" diff --git a/packages/cdk/resources/VectorKnowledgeBaseResources.ts b/packages/cdk/resources/VectorKnowledgeBaseResources.ts index f83e5a731..7bb0bd160 100644 --- a/packages/cdk/resources/VectorKnowledgeBaseResources.ts +++ b/packages/cdk/resources/VectorKnowledgeBaseResources.ts @@ -156,15 +156,13 @@ export class VectorKnowledgeBaseResources extends Construct { // Create S3 data source for knowledge base documents // prefix pointed to processed/ to only ingest converted markdown documents - const chunkingConfiguration = { + const chunkingConfiguration: CfnDataSource.ChunkingConfigurationProperty = { ...ChunkingStrategy.SEMANTIC.configuration, - hierarchicalChunkingConfiguration: { - overlapTokens: 60, - levelConfigurations: [ - {maxTokens: 1000}, // Parent chunk configuration, - {maxTokens: 300} // Child chunk configuration - ] - } + semanticChunkingConfiguration: { + breakpointPercentileThreshold: 80, + bufferSize: 1, + maxTokens: 350 + } satisfies CfnDataSource.SemanticChunkingConfigurationProperty } const hash = crypto.createHash("md5") diff --git a/packages/slackBotFunction/app/services/bedrock.py b/packages/slackBotFunction/app/services/bedrock.py index df86c3acd..f0b4e1c08 100644 --- a/packages/slackBotFunction/app/services/bedrock.py +++ b/packages/slackBotFunction/app/services/bedrock.py @@ -25,7 +25,7 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat inference_config = prompt_template.get("inference_config") if not inference_config: - default_values = {"temperature": 0, "maxTokens": 1500, "topP": 1} + default_values = {"temperature": 0, "maxTokens": 1024, "topP": 0.1} inference_config = default_values logger.warning( "No inference configuration found in prompt template; using default values", diff --git a/packages/slackBotFunction/app/services/prompt_loader.py b/packages/slackBotFunction/app/services/prompt_loader.py index 10d941fba..262fe9c2c 100644 --- a/packages/slackBotFunction/app/services/prompt_loader.py +++ b/packages/slackBotFunction/app/services/prompt_loader.py @@ -110,7 +110,7 @@ def load_prompt(prompt_name: str, prompt_version: str = None) -> dict: actual_version = response.get("version", "DRAFT") # Extract inference configuration with defaults - default_inference = {"temperature": 0, "topP": 1, "maxTokens": 1500} + default_inference = {"temperature": 0, "topP": 0.1, "maxTokens": 1024} model_id = variant.get("modelId", "") raw_inference = variant.get("inferenceConfiguration", {}) raw_text_config = raw_inference.get("text", {}) diff --git a/packages/slackBotFunction/tests/test_bedrock_integration.py b/packages/slackBotFunction/tests/test_bedrock_integration.py index 8b4877c7c..3ece533e7 100644 --- a/packages/slackBotFunction/tests/test_bedrock_integration.py +++ b/packages/slackBotFunction/tests/test_bedrock_integration.py @@ -114,7 +114,7 @@ def test_query_bedrock_check_config(mock_boto_client: Mock, mock_load_prompt: Mo mock_client.retrieve_and_generate.return_value = {"output": {"text": "response"}} mock_load_prompt.return_value = { "prompt_text": "Test prompt template", - "inference_config": {"temperature": "0", "maxTokens": "1500", "topP": "1"}, + "inference_config": {"temperature": "0", "maxTokens": "1024", "topP": "0.1"}, } # delete and import module to test From 8676458291d6449efb9be129b58b1857f2f4c68a Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Thu, 12 Feb 2026 17:08:34 +0000 Subject: [PATCH 04/21] feat: update tests for inference --- packages/slackBotFunction/tests/test_bedrock_integration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/slackBotFunction/tests/test_bedrock_integration.py b/packages/slackBotFunction/tests/test_bedrock_integration.py index 3ece533e7..2148cc9c8 100644 --- a/packages/slackBotFunction/tests/test_bedrock_integration.py +++ b/packages/slackBotFunction/tests/test_bedrock_integration.py @@ -132,5 +132,5 @@ def test_query_bedrock_check_config(mock_boto_client: Mock, mock_load_prompt: Mo ]["inferenceConfig"]["textInferenceConfig"] assert prompt_config["temperature"] == "0" - assert prompt_config["maxTokens"] == "1500" - assert prompt_config["topP"] == "1" + assert prompt_config["maxTokens"] == "1024" + assert prompt_config["topP"] == "0.1" From 4c6b6f8b147939086cd5b14e749a9ae0fd50a09b Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Fri, 13 Feb 2026 09:42:15 +0000 Subject: [PATCH 05/21] feat: Use reformulation prompt Add default reformulation prompt from Bedrock knowledgebase test --- packages/cdk/prompts/reformulationPrompt.txt | 40 +++++++++++++++++-- .../cdk/resources/BedrockPromptResources.ts | 2 +- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/cdk/prompts/reformulationPrompt.txt b/packages/cdk/prompts/reformulationPrompt.txt index 60b255819..7b8a2a804 100644 --- a/packages/cdk/prompts/reformulationPrompt.txt +++ b/packages/cdk/prompts/reformulationPrompt.txt @@ -1,5 +1,39 @@ -Return the user query exactly as provided without any modifications, changes, or reformulations. -Do not alter, rephrase, or modify the input in any way. -Simply return: {{user_query}} +You are a query creation agent. You will be provided with a function and a description of what it searches over. The user will provide you a question, and your job is to determine the optimal query to use based on the user's question. +Here are a few examples of queries formed by other search function selection and query creation agents: + + + + What if my vehicle is totaled in an accident? + what happens if my vehicle is totaled + + + I am relocating within the same state. Can I keep my current agent? + can I keep my current agent when moving in state + + + +You should also pay attention to the conversation history between the user and the search engine in order to gain the context necessary to create the query. +Here's another example that shows how you should reference the conversation history when generating a query: + + + + + How many vehicles can I include in a quote in Kansas + You can include 5 vehicles in a quote if you live in Kansas + + + What about texas? + You can include 3 vehicles in a quote if you live in Texas + + + + +IMPORTANT: the elements in the tags should not be assumed to have been provided to you to use UNLESS they are also explicitly given to you below. +All of the values and information within the examples (the questions, answers, and function calls) are strictly part of the examples and have not been provided to you. + +Here is the current conversation history: +$conversation_history$ User Query: {{user_query}} + +$output_format_instructions$ diff --git a/packages/cdk/resources/BedrockPromptResources.ts b/packages/cdk/resources/BedrockPromptResources.ts index 73a8fa2ce..a1ee367c2 100644 --- a/packages/cdk/resources/BedrockPromptResources.ts +++ b/packages/cdk/resources/BedrockPromptResources.ts @@ -21,7 +21,7 @@ export class BedrockPromptResources extends Construct { super(scope, id) const ragModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") - const reformulationModel = BedrockFoundationModel.AMAZON_NOVA_LITE_V1 + const reformulationModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") const queryReformulationPromptVariant = PromptVariant.text({ variantName: "default", From 1d6b1254022959e861b9d958ec4d6d6717865cdb Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Fri, 13 Feb 2026 10:32:10 +0000 Subject: [PATCH 06/21] feat: Replicate ai models for rag and reformulation --- packages/cdk/nagSuppressions.ts | 16 ++++ packages/cdk/resources/Apis.ts | 12 +++ .../cdk/resources/BedrockPromptResources.ts | 81 +++++++++++-------- .../cdk/resources/BedrockPromptSettings.ts | 14 +++- packages/cdk/stacks/EpsAssistMeStack.ts | 10 +-- 5 files changed, 92 insertions(+), 41 deletions(-) diff --git a/packages/cdk/nagSuppressions.ts b/packages/cdk/nagSuppressions.ts index 3de6806a5..5482bab4a 100644 --- a/packages/cdk/nagSuppressions.ts +++ b/packages/cdk/nagSuppressions.ts @@ -80,6 +80,22 @@ export const nagSuppressions = (stack: Stack, account: string) => { ] ) + // Suppress unauthenticated API route warnings + safeAddNagSuppression( + stack, + "/EpsAssistMeStack/Apis/EpsAssistApiGateway/ApiGateway/Default/slack/commands/POST/Resource", + [ + { + id: "AwsSolutions-APIG4", + reason: "Slack command endpoint is intentionally unauthenticated." + }, + { + id: "AwsSolutions-COG4", + reason: "Cognito not required for this public endpoint." + } + ] + ) + // Suppress missing WAF on API stage for Apis construct safeAddNagSuppression( stack, diff --git a/packages/cdk/resources/Apis.ts b/packages/cdk/resources/Apis.ts index c67c9fba3..fd8d637db 100644 --- a/packages/cdk/resources/Apis.ts +++ b/packages/cdk/resources/Apis.ts @@ -27,6 +27,7 @@ export class Apis extends Construct { forwardCsocLogs: props.forwardCsocLogs, csocApiGatewayDestination: props.csocApiGatewayDestination }) + // Create /slack resource path const slackResource = apiGateway.api.root.addResource("slack") @@ -41,6 +42,17 @@ export class Apis extends Construct { lambdaFunction: props.functions.slackBot }) + // Create the '/slack/commands' POST endpoint for Slack Events API + // This endpoint will handle slash commands, such as /test + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const slackCommandsEndpoint = new LambdaEndpoint(this, "SlackCommandsEndpoint", { + parentResource: slackResource, + resourceName: "commands", + method: HttpMethod.POST, + restApiGatewayRole: apiGateway.role, + lambdaFunction: props.functions.slackBot + }) + this.apis = { api: apiGateway } diff --git a/packages/cdk/resources/BedrockPromptResources.ts b/packages/cdk/resources/BedrockPromptResources.ts index a1ee367c2..a756e2535 100644 --- a/packages/cdk/resources/BedrockPromptResources.ts +++ b/packages/cdk/resources/BedrockPromptResources.ts @@ -1,10 +1,12 @@ import {Construct} from "constructs" import { BedrockFoundationModel, + ChatMessage, Prompt, PromptVariant } from "@cdklabs/generative-ai-cdk-constructs/lib/cdk-lib/bedrock" import {BedrockPromptSettings} from "./BedrockPromptSettings" +import {CfnPrompt} from "aws-cdk-lib/aws-bedrock" export interface BedrockPromptResourcesProps { readonly stackName: string @@ -14,53 +16,64 @@ export interface BedrockPromptResourcesProps { export class BedrockPromptResources extends Construct { public readonly queryReformulationPrompt: Prompt public readonly ragResponsePrompt: Prompt - public readonly ragModelId: string - public readonly queryReformulationModelId: string + public readonly modelId: string constructor(scope: Construct, id: string, props: BedrockPromptResourcesProps) { super(scope, id) - const ragModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") - const reformulationModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") + const aiModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") - const queryReformulationPromptVariant = PromptVariant.text({ - variantName: "default", - model: reformulationModel, - promptVariables: ["topic"], - promptText: props.settings.reformulationPrompt.text - }) + // Create Prompts + this.queryReformulationPrompt = this.createPrompt( + "QueryReformulationPrompt", + `${props.stackName}-queryReformulation`, + "Prompt for reformulating user queries to improve RAG retrieval", + aiModel, + "", + [props.settings.reformulationPrompt], + props.settings.reformulationInferenceConfig + ) - const queryReformulationPrompt = new Prompt(this, "QueryReformulationPrompt", { - promptName: `${props.stackName}-queryReformulation`, - description: "Prompt for reformulating user queries to improve RAG retrieval", - defaultVariant: queryReformulationPromptVariant, - variants: [queryReformulationPromptVariant] - }) + this.ragResponsePrompt = this.createPrompt( + "RagResponsePrompt", + `${props.stackName}-ragResponse`, + "Prompt for generating RAG responses with knowledge base context and system instructions", + aiModel, + props.settings.systemPrompt.text, + [props.settings.userPrompt], + props.settings.ragInferenceConfig + ) + + this.modelId = aiModel.modelId + } + + private createPrompt( + id: string, + promptName: string, + description: string, + model: BedrockFoundationModel, + systemPromptText: string, + messages: [ChatMessage], + inferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty + ): Prompt { - const ragResponsePromptVariant = PromptVariant.chat({ + const variant = PromptVariant.chat({ variantName: "default", - model: ragModel, + model: model, promptVariables: ["query", "search_results"], - system: props.settings.systemPrompt.text, - messages: [props.settings.userPrompt] + system: systemPromptText, + messages: messages }) - ragResponsePromptVariant.inferenceConfiguration = { - text: props.settings.inferenceConfig + variant.inferenceConfiguration = { + text: inferenceConfig } - const ragPrompt = new Prompt(this, "ragResponsePrompt", { - promptName: `${props.stackName}-ragResponse`, - description: "Prompt for generating RAG responses with knowledge base context and system instructions", - defaultVariant: ragResponsePromptVariant, - variants: [ragResponsePromptVariant] + return new Prompt(this, id, { + promptName, + description, + defaultVariant: variant, + variants: [variant] }) - - // expose model IDs for use in Lambda environment variables - this.ragModelId = ragModel.modelId - this.queryReformulationModelId = reformulationModel.modelId - - this.queryReformulationPrompt = queryReformulationPrompt - this.ragResponsePrompt = ragPrompt } } diff --git a/packages/cdk/resources/BedrockPromptSettings.ts b/packages/cdk/resources/BedrockPromptSettings.ts index 3cd518f4d..0b45f14ab 100644 --- a/packages/cdk/resources/BedrockPromptSettings.ts +++ b/packages/cdk/resources/BedrockPromptSettings.ts @@ -13,7 +13,8 @@ export class BedrockPromptSettings extends Construct { public readonly systemPrompt: ChatMessage public readonly userPrompt: ChatMessage public readonly reformulationPrompt: ChatMessage - public readonly inferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty + public readonly ragInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty + public readonly reformulationInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty /** * @param scope The Construct scope @@ -32,7 +33,7 @@ export class BedrockPromptSettings extends Construct { const reformulationPrompt = this.getTypedPrompt("reformulation") this.reformulationPrompt = ChatMessage.user(reformulationPrompt.text) - this.inferenceConfig = { + this.ragInferenceConfig = { temperature: 0, topP: 0.1, maxTokens: 1024, @@ -40,6 +41,15 @@ export class BedrockPromptSettings extends Construct { "Human:" ] } + + this.reformulationInferenceConfig = { + temperature: 0.5, + topP: 0.9, + maxTokens: 512, + stopSequences: [ + "Human:" + ] + } } /** Get the latest prompt text from files in the specified directory. diff --git a/packages/cdk/stacks/EpsAssistMeStack.ts b/packages/cdk/stacks/EpsAssistMeStack.ts index cdf1df80f..42f871cce 100644 --- a/packages/cdk/stacks/EpsAssistMeStack.ts +++ b/packages/cdk/stacks/EpsAssistMeStack.ts @@ -164,8 +164,8 @@ export class EpsAssistMeStack extends Stack { guardrailArn: vectorKB.guardrail.guardrailArn, dataSourceArn: vectorKB.dataSourceArn, promptName: bedrockPromptResources.queryReformulationPrompt.promptName, - ragModelId: bedrockPromptResources.ragModelId, - queryReformulationModelId: bedrockPromptResources.queryReformulationModelId, + ragModelId: bedrockPromptResources.modelId, + queryReformulationModelId: bedrockPromptResources.modelId, docsBucketArn: storage.kbDocsBucket.bucketArn, docsBucketKmsKeyArn: storage.kbDocsKmsKey.keyArn }) @@ -196,8 +196,8 @@ export class EpsAssistMeStack extends Stack { ragResponsePromptName: bedrockPromptResources.ragResponsePrompt.promptName, reformulationPromptVersion: bedrockPromptResources.queryReformulationPrompt.promptVersion, ragResponsePromptVersion: bedrockPromptResources.ragResponsePrompt.promptVersion, - ragModelId: bedrockPromptResources.ragModelId, - queryReformulationModelId: bedrockPromptResources.queryReformulationModelId, + ragModelId: bedrockPromptResources.modelId, + queryReformulationModelId: bedrockPromptResources.modelId, isPullRequest: isPullRequest, mainSlackBotLambdaExecutionRoleArn: mainSlackBotLambdaExecutionRoleArn, notifyS3UploadFunctionPolicy: runtimePolicies.notifyS3UploadFunctionPolicy, @@ -276,7 +276,7 @@ export class EpsAssistMeStack extends Stack { // Output: SlackBot Endpoint new CfnOutput(this, "SlackBotCommandsEndpoint", { value: `https://${apis.apis["api"].api.domainName?.domainName}/slack/commands`, - description: "Slack Commands API endpoint for slash commands" + description: "Slack Commands API endpoint for /slash commands" }) // Output: Bedrock Prompt ARN From abe3054e12efe13b608b5197675d1cc1c277235a Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Fri, 13 Feb 2026 11:05:06 +0000 Subject: [PATCH 07/21] feat: Replicate ai models for rag and reformulation --- packages/cdk/prompts/systemPrompt.txt | 44 ++++++++++--------- .../cdk/resources/BedrockPromptResources.ts | 8 +++- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/cdk/prompts/systemPrompt.txt b/packages/cdk/prompts/systemPrompt.txt index b058420cb..d29b9b3bc 100644 --- a/packages/cdk/prompts/systemPrompt.txt +++ b/packages/cdk/prompts/systemPrompt.txt @@ -1,24 +1,28 @@ -# 1. Persona & Logic -You are an AI assistant for onboarding guidance. Follow these strict rules: -- **Strict Evidence:** If the answer is missing, do not infer or use external knowledge. -- **Grounding:** NEVER use your own internal training data, online resources, or prior knowledge. -- **Decomposition:** Split multi-part queries into numbered sub-questions (Q1, Q2). +You are a technical assistant specialized in onboarding guidance. +Your primary goal is to answer questions using ONLY the provided search results. -# 2. Output Structure -**Summary** -2-3 sentences maximum. +STYLE & FORMATTING RULES: +- Do NOT refer to the search results by number or name in the body of the text. +- Do NOT add a "Citations" section at the end of the response. +- Do NOT reference how the information was found (e.g., "...the provided search results") +- Text should prioritie readability. +- Links should use Markdown text, e.g., [link text](url). +- Use `Inline Code` for system names, field names, or technical terms (e.g., `HL7 FHIR`). - **Answer** - Prioritize detail and specification, focus on the information direct at the question. +STEPS: +1. Generate an answer, capturing the core question the user is asking. +2. Answer, directly, any individual or sub-questions the user has provided. +3. You must create a very short summary encapsulating the response and have it precede all other answers. -# 3. Styling Rules (`mrkdwn`) -Use British English. -- **Bold (`*`):** Headings, Subheadings, Source Names, and important information/ exceptions (e.g. `*NHS England*`). -- **Italic (`_`):** Citations and Titles (e.g. `_Guidance v1_`). -- **Blockquote (`>`):** Quotes (>1 sentence) and Tech Specs/Examples (e.g. `HL7 FHIR`). -- **Links:** `[text](link)`. +EXAMPLE: + +*Summary* +This is a short answer the captures the core question provided. -# 4. Format Rules -- NEVER use in-line references or citations (e.g., do not write "(search result 1)" or "[1]"). -- Do NOT refer to the search results by number or name in the body of the text. -- Do NOT add a "Citations" section at the end of the response.wer, details from the knowledge base. +*Answer* +This is a direct answer to the question, or questions, provided. It is in-depth, and breaks down individual questions. There is no reference to the text here (for example, you don't see "from source 1") but instead treats this information as if it was public knowledge. However, if there is a source, it does provide that source [as a hyperlink](hyperlink) to the website it can be found. + +There is multiple paragraphs, with blank lines between, to make it easier to read, as readability is a requirement. + +For more details, please refer to the [Authentication Guide](http://example.com/guide). + diff --git a/packages/cdk/resources/BedrockPromptResources.ts b/packages/cdk/resources/BedrockPromptResources.ts index a756e2535..02b75474b 100644 --- a/packages/cdk/resources/BedrockPromptResources.ts +++ b/packages/cdk/resources/BedrockPromptResources.ts @@ -1,4 +1,5 @@ import {Construct} from "constructs" +import * as crypto from "crypto" import { BedrockFoundationModel, ChatMessage, @@ -69,8 +70,13 @@ export class BedrockPromptResources extends Construct { text: inferenceConfig } + const hash = crypto.createHash("md5") + .update(JSON.stringify(variant)) + .digest("hex") + .substring(0, 6) + return new Prompt(this, id, { - promptName, + promptName: `${promptName}-${hash}`, description, defaultVariant: variant, variants: [variant] From fb955b3c549277d01656dee1b2de5cb6333c1974 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Fri, 13 Feb 2026 12:07:36 +0000 Subject: [PATCH 08/21] feat: Move prompt reformulation to rag orchestration --- ...tionPrompt.txt => orchestrationPrompt.txt} | 2 - .../cdk/resources/BedrockPromptResources.ts | 12 +-- packages/cdk/resources/Functions.ts | 8 +- packages/cdk/stacks/EpsAssistMeStack.ts | 4 +- packages/slackBotFunction/app/core/config.py | 6 ++ .../app/services/ai_processor.py | 6 +- .../slackBotFunction/app/services/bedrock.py | 33 ++++++- .../app/services/prompt_loader.py | 20 ++-- .../app/services/query_reformulator.py | 83 ---------------- packages/slackBotFunction/tests/conftest.py | 4 +- .../tests/test_ai_processor.py | 48 ++-------- .../tests/test_bedrock_integration.py | 8 +- .../tests/test_prompt_loader.py | 36 +++---- .../tests/test_query_reformulator.py | 94 ------------------- 14 files changed, 91 insertions(+), 273 deletions(-) rename packages/cdk/prompts/{reformulationPrompt.txt => orchestrationPrompt.txt} (98%) delete mode 100644 packages/slackBotFunction/app/services/query_reformulator.py delete mode 100644 packages/slackBotFunction/tests/test_query_reformulator.py diff --git a/packages/cdk/prompts/reformulationPrompt.txt b/packages/cdk/prompts/orchestrationPrompt.txt similarity index 98% rename from packages/cdk/prompts/reformulationPrompt.txt rename to packages/cdk/prompts/orchestrationPrompt.txt index 7b8a2a804..acecd43ef 100644 --- a/packages/cdk/prompts/reformulationPrompt.txt +++ b/packages/cdk/prompts/orchestrationPrompt.txt @@ -34,6 +34,4 @@ All of the values and information within the examples (the questions, answers, a Here is the current conversation history: $conversation_history$ -User Query: {{user_query}} - $output_format_instructions$ diff --git a/packages/cdk/resources/BedrockPromptResources.ts b/packages/cdk/resources/BedrockPromptResources.ts index 02b75474b..077c1f708 100644 --- a/packages/cdk/resources/BedrockPromptResources.ts +++ b/packages/cdk/resources/BedrockPromptResources.ts @@ -15,7 +15,7 @@ export interface BedrockPromptResourcesProps { } export class BedrockPromptResources extends Construct { - public readonly queryReformulationPrompt: Prompt + public readonly orchestrationReformulationPrompt: Prompt public readonly ragResponsePrompt: Prompt public readonly modelId: string @@ -25,10 +25,10 @@ export class BedrockPromptResources extends Construct { const aiModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") // Create Prompts - this.queryReformulationPrompt = this.createPrompt( - "QueryReformulationPrompt", - `${props.stackName}-queryReformulation`, - "Prompt for reformulating user queries to improve RAG retrieval", + this.orchestrationReformulationPrompt = this.createPrompt( + "OrchestrationReformulationPrompt", + `${props.stackName}-OrchestrationReformulation`, + "Prompt for orchestrating queries to improve RAG inference", aiModel, "", [props.settings.reformulationPrompt], @@ -61,7 +61,7 @@ export class BedrockPromptResources extends Construct { const variant = PromptVariant.chat({ variantName: "default", model: model, - promptVariables: ["query", "search_results"], + promptVariables: ["prompt", "search_results"], system: systemPromptText, messages: messages }) diff --git a/packages/cdk/resources/Functions.ts b/packages/cdk/resources/Functions.ts index 322447981..50557f9bf 100644 --- a/packages/cdk/resources/Functions.ts +++ b/packages/cdk/resources/Functions.ts @@ -28,9 +28,9 @@ export interface FunctionsProps { readonly slackBotTokenSecret: Secret readonly slackBotSigningSecret: Secret readonly slackBotStateTable: TableV2 - readonly reformulationPromptName: string + readonly orchestrationPromptName: string readonly ragResponsePromptName: string - readonly reformulationPromptVersion: string + readonly orchestrationPromptVersion: string readonly ragResponsePromptVersion: string readonly isPullRequest: boolean readonly mainSlackBotLambdaExecutionRoleArn : string @@ -69,9 +69,9 @@ export class Functions extends Construct { "GUARD_RAIL_ID": props.guardrailId, "GUARD_RAIL_VERSION": props.guardrailVersion, "SLACK_BOT_STATE_TABLE": props.slackBotStateTable.tableName, - "QUERY_REFORMULATION_PROMPT_NAME": props.reformulationPromptName, + "ORCHESTRATION_RESPONSE_PROMPT_NAME": props.orchestrationPromptName, "RAG_RESPONSE_PROMPT_NAME": props.ragResponsePromptName, - "QUERY_REFORMULATION_PROMPT_VERSION": props.reformulationPromptVersion, + "ORCHESTRATION_RESPONSE_PROMPT_VERSION": props.orchestrationPromptVersion, "RAG_RESPONSE_PROMPT_VERSION": props.ragResponsePromptVersion } }) diff --git a/packages/cdk/stacks/EpsAssistMeStack.ts b/packages/cdk/stacks/EpsAssistMeStack.ts index 42f871cce..62174ffc1 100644 --- a/packages/cdk/stacks/EpsAssistMeStack.ts +++ b/packages/cdk/stacks/EpsAssistMeStack.ts @@ -192,9 +192,9 @@ export class EpsAssistMeStack extends Stack { slackBotTokenSecret: secrets.slackBotTokenSecret, slackBotSigningSecret: secrets.slackBotSigningSecret, slackBotStateTable: tables.slackBotStateTable.table, - reformulationPromptName: bedrockPromptResources.queryReformulationPrompt.promptName, + orchestrationPromptName: bedrockPromptResources.queryReformulationPrompt.promptName, ragResponsePromptName: bedrockPromptResources.ragResponsePrompt.promptName, - reformulationPromptVersion: bedrockPromptResources.queryReformulationPrompt.promptVersion, + orchestrationPromptVersion: bedrockPromptResources.queryReformulationPrompt.promptVersion, ragResponsePromptVersion: bedrockPromptResources.ragResponsePrompt.promptVersion, ragModelId: bedrockPromptResources.modelId, queryReformulationModelId: bedrockPromptResources.modelId, diff --git a/packages/slackBotFunction/app/core/config.py b/packages/slackBotFunction/app/core/config.py index 6e980c55b..146a0f5d3 100644 --- a/packages/slackBotFunction/app/core/config.py +++ b/packages/slackBotFunction/app/core/config.py @@ -81,6 +81,8 @@ def get_retrieve_generate_config() -> BedrockConfig: GUARD_VERSION = os.environ["GUARD_RAIL_VERSION"] RAG_RESPONSE_PROMPT_NAME = os.environ["RAG_RESPONSE_PROMPT_NAME"] RAG_RESPONSE_PROMPT_VERSION = os.environ["RAG_RESPONSE_PROMPT_VERSION"] + ORCHESTRATION_RESPONSE_PROMPT_NAME = os.environ["ORCHESTRATION_RESPONSE_PROMPT_NAME"] + ORCHESTRATION_RESPONSE_PROMPT_VERSION = os.environ["ORCHESTRATION_RESPONSE_PROMPT_VERSION"] logger.info( "Guardrail configuration loaded", extra={"guardrail_id": GUARD_RAIL_ID, "guardrail_version": GUARD_VERSION} @@ -94,6 +96,8 @@ def get_retrieve_generate_config() -> BedrockConfig: GUARD_VERSION, RAG_RESPONSE_PROMPT_NAME, RAG_RESPONSE_PROMPT_VERSION, + ORCHESTRATION_RESPONSE_PROMPT_NAME, + ORCHESTRATION_RESPONSE_PROMPT_VERSION, ) @@ -148,6 +152,8 @@ class BedrockConfig: GUARD_VERSION: str RAG_RESPONSE_PROMPT_NAME: str RAG_RESPONSE_PROMPT_VERSION: str + ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME: str + ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_VERSION: str @dataclass diff --git a/packages/slackBotFunction/app/services/ai_processor.py b/packages/slackBotFunction/app/services/ai_processor.py index 857a9e24e..fab30e966 100644 --- a/packages/slackBotFunction/app/services/ai_processor.py +++ b/packages/slackBotFunction/app/services/ai_processor.py @@ -6,7 +6,6 @@ """ from app.services.bedrock import query_bedrock -from app.services.query_reformulator import reformulate_query from app.core.config import get_logger from app.core.types import AIProcessorResponse @@ -15,11 +14,8 @@ def process_ai_query(user_query: str, session_id: str | None = None) -> AIProcessorResponse: """shared AI processing logic for both slack and direct invocation""" - # reformulate: improves vector search quality in knowledge base - reformulated_query = reformulate_query(user_query) - # session_id enables conversation continuity across multiple queries - kb_response = query_bedrock(reformulated_query, session_id) + kb_response = query_bedrock(user_query, session_id) logger.info( "response from bedrock", diff --git a/packages/slackBotFunction/app/services/bedrock.py b/packages/slackBotFunction/app/services/bedrock.py index f0b4e1c08..ec5305952 100644 --- a/packages/slackBotFunction/app/services/bedrock.py +++ b/packages/slackBotFunction/app/services/bedrock.py @@ -21,8 +21,12 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat """ config = get_retrieve_generate_config() - prompt_template = load_prompt(config.RAG_RESPONSE_PROMPT_NAME, config.RAG_RESPONSE_PROMPT_VERSION) - inference_config = prompt_template.get("inference_config") + rag_prompt_template = load_prompt(config.RAG_RESPONSE_PROMPT_NAME, config.RAG_RESPONSE_PROMPT_VERSION) + orchestration_prompt_template = load_prompt( + config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME, + config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_VERSION, + ) + inference_config = rag_prompt_template.get("inference_config") if not inference_config: default_values = {"temperature": 0, "maxTokens": 1024, "topP": 0.1} @@ -42,7 +46,7 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat "type": "KNOWLEDGE_BASE", "knowledgeBaseConfiguration": { "knowledgeBaseId": config.KNOWLEDGEBASE_ID, - "modelArn": prompt_template.get("model_id", config.RAG_MODEL_ID), + "modelArn": rag_prompt_template.get("model_id", config.RAG_MODEL_ID), "retrievalConfiguration": {"vectorSearchConfiguration": {"numberOfResults": 5}}, "generationConfiguration": { "guardrailConfiguration": { @@ -59,17 +63,36 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat }, }, }, + "orchestrationConfiguration": { + "inferenceConfig": { + "textInferenceConfig": { + **inference_config, + "stopSequences": [ + "Human:", + ], + } + }, + }, }, } - if prompt_template: + if rag_prompt_template: request_params["retrieveAndGenerateConfiguration"]["knowledgeBaseConfiguration"]["generationConfiguration"][ "promptTemplate" - ] = {"textPromptTemplate": prompt_template.get("prompt_text")} + ] = {"textPromptTemplate": rag_prompt_template.get("prompt_text")} logger.info( "Using prompt template for RAG response generation", extra={"prompt_name": config.RAG_RESPONSE_PROMPT_NAME} ) + if orchestration_prompt_template: + request_params["retrieveAndGenerateConfiguration"]["orchestrationConfiguration"]["promptTemplate"] = { + "textPromptTemplate": orchestration_prompt_template.get("prompt_text") + } + logger.info( + "Using prompt template for RAG response generation", + extra={"prompt_name": config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME}, + ) + # Include session ID for conversation continuity across messages if session_id: request_params["sessionId"] = session_id diff --git a/packages/slackBotFunction/app/services/prompt_loader.py b/packages/slackBotFunction/app/services/prompt_loader.py index 262fe9c2c..cf1bebded 100644 --- a/packages/slackBotFunction/app/services/prompt_loader.py +++ b/packages/slackBotFunction/app/services/prompt_loader.py @@ -9,14 +9,14 @@ logger = get_logger() -def _render_prompt(template_config: dict) -> str: +def _render_system_prompt(template_config: dict) -> str: """ Returns a unified prompt string regardless of template type. """ - chat_cfg = template_config.get("chat") - if chat_cfg: - return parse_system_message(chat_cfg) + chat_configuration = template_config.get("chat") + if chat_configuration: + return parse_system_message(chat_configuration) text_cfg = template_config.get("text") if isinstance(text_cfg, dict) and "text" in text_cfg: @@ -31,10 +31,10 @@ def _render_prompt(template_config: dict) -> str: raise PromptLoadError(f"Unsupported prompt configuration. Keys: {list(template_config.keys())}") -def parse_system_message(chat_cfg: dict) -> str: +def parse_system_message(chat_configuration: dict) -> str: parts: list[str] = [] - system_items = chat_cfg.get("system", []) + system_items = chat_configuration.get("system", []) logger.debug("Processing system messages for prompt rendering", extra={"system_items": system_items}) if isinstance(system_items, list): system_texts = [ @@ -50,9 +50,11 @@ def parse_system_message(chat_cfg: dict) -> str: "assistant": "Assistant: ", } - logger.debug("Processing chat messages for prompt rendering", extra={"messages": chat_cfg.get("messages", [])}) + logger.debug( + "Processing chat messages for prompt rendering", extra={"messages": chat_configuration.get("messages", [])} + ) - for msg in chat_cfg.get("messages", []): + for msg in chat_configuration.get("messages", []): role = (msg.get("role") or "").lower() prefix = role_prefix.get(role) if not prefix: @@ -106,7 +108,7 @@ def load_prompt(prompt_name: str, prompt_version: str = None) -> dict: # Extract and render the prompt template template_config = variant["templateConfiguration"] - prompt_text = _render_prompt(template_config) + prompt_text = _render_system_prompt(template_config) actual_version = response.get("version", "DRAFT") # Extract inference configuration with defaults diff --git a/packages/slackBotFunction/app/services/query_reformulator.py b/packages/slackBotFunction/app/services/query_reformulator.py deleted file mode 100644 index 4a2bd060e..000000000 --- a/packages/slackBotFunction/app/services/query_reformulator.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import traceback -import boto3 - -from app.core.config import get_logger -from app.services.bedrock import invoke_model -from .prompt_loader import load_prompt -from .exceptions import ConfigurationError -from mypy_boto3_bedrock_runtime.client import BedrockRuntimeClient - -logger = get_logger() - - -def reformulate_query(user_query: str) -> str: - """ - Reformulate user query using Amazon Nova Lite for better RAG retrieval. - - Loads prompt template from Bedrock Prompt Management, formats it with the user's - query, and uses Nova Lite to generate a reformulated version optimized for vector search. - """ - try: - client: BedrockRuntimeClient = boto3.client("bedrock-runtime", region_name=os.environ["AWS_REGION"]) - model_id = os.environ["QUERY_REFORMULATION_MODEL_ID"] - - # Load prompt template from Bedrock Prompt Management - prompt_name = os.environ.get("QUERY_REFORMULATION_PROMPT_NAME") - prompt_version = os.environ.get("QUERY_REFORMULATION_PROMPT_VERSION", "DRAFT") - - if not prompt_name: - raise ConfigurationError("QUERY_REFORMULATION_PROMPT_NAME environment variable not set") - - # Load prompt with specified version (DRAFT by default) - prompt_template = load_prompt(prompt_name, prompt_version) - - logger.info( - "Prompt loaded successfully from Bedrock", - extra={"prompt_name": prompt_name, "version_used": prompt_version}, - ) - - # Format the prompt with the user query (using double braces from Bedrock template) - prompt = prompt_template.get("prompt_text").replace("{{user_query}}", user_query) - result = invoke_model( - prompt=prompt, model_id=model_id, client=client, inference_config=prompt_template.get("inference_config") - ) - - reformulated_query = result["content"][0]["text"].strip() - - logger.info( - "Query reformulated successfully using Bedrock prompt", - extra={ - "original_query": user_query, - "reformulated_query": reformulated_query, - "prompt_version_used": prompt_version, - "prompt_source": "bedrock_prompt_management", - }, - ) - - return reformulated_query - - except Exception as e: - logger.error( - f"Failed to reformulate query using Bedrock prompts: {e}", - extra={ - "original_query": user_query, - "prompt_name": os.environ.get("QUERY_REFORMULATION_PROMPT_NAME"), - "prompt_version": os.environ.get("QUERY_REFORMULATION_PROMPT_VERSION", "auto"), - "error_type": type(e).__name__, - "error": traceback.format_exc(), - }, - ) - - # Graceful degradation - return original query but alert on infrastructure issue - logger.error( - "Query reformulation degraded: Bedrock Prompt Management unavailable", - extra={ - "service_status": "degraded", - "fallback_action": "using_original_query", - "requires_attention": True, - "impact": "reduced_rag_quality", - }, - ) - - return user_query # Minimal fallback - just return original query diff --git a/packages/slackBotFunction/tests/conftest.py b/packages/slackBotFunction/tests/conftest.py index 3b62aecc8..7271b67f0 100644 --- a/packages/slackBotFunction/tests/conftest.py +++ b/packages/slackBotFunction/tests/conftest.py @@ -22,8 +22,8 @@ def mock_env(): "GUARD_RAIL_ID": "test-guard-id", "GUARD_RAIL_VERSION": "1", "QUERY_REFORMULATION_MODEL_ID": "test-model", - "QUERY_REFORMULATION_PROMPT_NAME": "test-prompt", - "QUERY_REFORMULATION_PROMPT_VERSION": "DRAFT", + "ORCHESTRATION_RESPONSE_PROMPT_NAME": "test-prompt", + "ORCHESTRATION_RESPONSE_PROMPT_VERSION": "DRAFT", "RAG_RESPONSE_PROMPT_NAME": "test-rag-prompt", "RAG_RESPONSE_PROMPT_VERSION": "DRAFT", } diff --git a/packages/slackBotFunction/tests/test_ai_processor.py b/packages/slackBotFunction/tests/test_ai_processor.py index 8cb9f7fbe..66249e89e 100644 --- a/packages/slackBotFunction/tests/test_ai_processor.py +++ b/packages/slackBotFunction/tests/test_ai_processor.py @@ -8,10 +8,8 @@ class TestAIProcessor: @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_without_session(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_without_session(self, mock_bedrock): """new conversation: no session context passed to bedrock""" - mock_reformulate.return_value = "reformulated: How to authenticate EPS API?" mock_bedrock.return_value = { "output": {"text": "To authenticate with EPS API, you need..."}, "sessionId": "new-session-abc123", @@ -26,14 +24,11 @@ def test_process_ai_query_without_session(self, mock_reformulate, mock_bedrock): assert result["citations"][0]["title"] == "EPS Authentication Guide" assert "kb_response" in result - mock_reformulate.assert_called_once_with("How to authenticate EPS API?") - mock_bedrock.assert_called_once_with("reformulated: How to authenticate EPS API?", None) + mock_bedrock.assert_called_once_with("How to authenticate EPS API?", None) @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_with_session(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_with_session(self, mock_bedrock): """conversation continuity: existing session maintained across queries""" - mock_reformulate.return_value = "reformulated: What about rate limits?" mock_bedrock.return_value = { "output": {"text": "EPS API has rate limits of..."}, "sessionId": "existing-session-456", @@ -47,39 +42,21 @@ def test_process_ai_query_with_session(self, mock_reformulate, mock_bedrock): assert result["citations"] == [] assert "kb_response" in result - mock_reformulate.assert_called_once_with("What about rate limits?") - mock_bedrock.assert_called_once_with("reformulated: What about rate limits?", "existing-session-456") + mock_bedrock.assert_called_once_with("What about rate limits?", "existing-session-456") @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_reformulate_error(self, mock_reformulate, mock_bedrock): - """graceful degradation: reformulation failure bubbles up""" - mock_reformulate.side_effect = Exception("Query reformulation failed") - - with pytest.raises(Exception) as exc_info: - process_ai_query("How to authenticate EPS API?") - - assert "Query reformulation failed" in str(exc_info.value) - mock_bedrock.assert_not_called() - - @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_bedrock_error(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_bedrock_error(self, mock_bedrock): """bedrock service failure: error propagated to caller""" - mock_reformulate.return_value = "reformulated query" mock_bedrock.side_effect = Exception("Bedrock service error") with pytest.raises(Exception) as exc_info: process_ai_query("How to authenticate EPS API?") assert "Bedrock service error" in str(exc_info.value) - mock_reformulate.assert_called_once() @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_missing_citations(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_missing_citations(self, mock_bedrock): """bedrock response incomplete: citations default to empty list""" - mock_reformulate.return_value = "reformulated query" mock_bedrock.return_value = { "output": {"text": "Response without citations"}, "sessionId": "session-123", @@ -93,10 +70,8 @@ def test_process_ai_query_missing_citations(self, mock_reformulate, mock_bedrock assert result["citations"] == [] # safe default when bedrock omits citations @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_missing_session_id(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_missing_session_id(self, mock_bedrock): """bedrock response incomplete: session_id properly handles None""" - mock_reformulate.return_value = "reformulated query" mock_bedrock.return_value = { "output": {"text": "Response without session"}, "citations": [], @@ -110,10 +85,8 @@ def test_process_ai_query_missing_session_id(self, mock_reformulate, mock_bedroc assert result["citations"] == [] @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_empty_query(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_empty_query(self, mock_bedrock): """edge case: empty query still processed through full pipeline""" - mock_reformulate.return_value = "" mock_bedrock.return_value = { "output": {"text": "Please provide a question"}, "sessionId": "session-empty", @@ -123,14 +96,11 @@ def test_process_ai_query_empty_query(self, mock_reformulate, mock_bedrock): result = process_ai_query("") assert result["text"] == "Please provide a question" - mock_reformulate.assert_called_once_with("") mock_bedrock.assert_called_once_with("", None) @patch("app.services.ai_processor.query_bedrock") - @patch("app.services.ai_processor.reformulate_query") - def test_process_ai_query_includes_raw_response(self, mock_reformulate, mock_bedrock): + def test_process_ai_query_includes_raw_response(self, mock_bedrock): """slack needs raw bedrock data: kb_response preserved for session handling""" - mock_reformulate.return_value = "reformulated query" raw_response = { "output": {"text": "Test response"}, "sessionId": "test-123", diff --git a/packages/slackBotFunction/tests/test_bedrock_integration.py b/packages/slackBotFunction/tests/test_bedrock_integration.py index 2148cc9c8..a7f752de5 100644 --- a/packages/slackBotFunction/tests/test_bedrock_integration.py +++ b/packages/slackBotFunction/tests/test_bedrock_integration.py @@ -20,7 +20,7 @@ def test_get_bedrock_knowledgebase_response(mock_boto_client: Mock, mock_load_pr result = query_bedrock("test query") # assertions - mock_load_prompt.assert_called_once_with("test-rag-prompt", "DRAFT") + mock_load_prompt.assert_called_with("test-prompt", "DRAFT") mock_boto_client.assert_called_once_with(service_name="bedrock-agent-runtime", region_name="eu-west-2") mock_client.retrieve_and_generate.assert_called_once() assert result["output"]["text"] == "bedrock response" @@ -45,7 +45,7 @@ def test_query_bedrock_with_session(mock_boto_client: Mock, mock_load_prompt: Mo result = query_bedrock("test query", session_id="existing_session") # assertions - mock_load_prompt.assert_called_once_with("test-rag-prompt", "DRAFT") + mock_load_prompt.assert_called_with("test-prompt", "DRAFT") assert result == mock_response call_args = mock_client.retrieve_and_generate.call_args[1] assert call_args["sessionId"] == "existing_session" @@ -70,7 +70,7 @@ def test_query_bedrock_without_session(mock_boto_client: Mock, mock_load_prompt: result = query_bedrock("test query") # assertions - mock_load_prompt.assert_called_once_with("test-rag-prompt", "DRAFT") + mock_load_prompt.assert_called_with("test-prompt", "DRAFT") assert result == mock_response call_args = mock_client.retrieve_and_generate.call_args[1] assert "sessionId" not in call_args @@ -95,7 +95,7 @@ def test_query_bedrock_check_prompt(mock_boto_client: Mock, mock_load_prompt: Mo result = query_bedrock("test query") # assertions - mock_load_prompt.assert_called_once_with("test-rag-prompt", "DRAFT") + mock_load_prompt.assert_called_with("test-prompt", "DRAFT") call_args = mock_client.retrieve_and_generate.call_args[1] prompt_template = call_args["retrieveAndGenerateConfiguration"]["knowledgeBaseConfiguration"][ "generationConfiguration" diff --git a/packages/slackBotFunction/tests/test_prompt_loader.py b/packages/slackBotFunction/tests/test_prompt_loader.py index c86d8efd7..9733b85c4 100644 --- a/packages/slackBotFunction/tests/test_prompt_loader.py +++ b/packages/slackBotFunction/tests/test_prompt_loader.py @@ -158,9 +158,9 @@ def test_get_render_prompt_chat_dict(mock_logger: Mock, mock_env: Mock): # delete and import module to test if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt - result = _render_prompt( + result = _render_system_prompt( { "chat": { "system": [ @@ -186,9 +186,9 @@ def test_get_render_prompt_chat_dict_no_role(mock_logger: Mock, mock_env: Mock): # delete and import module to test if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt - result = _render_prompt( + result = _render_system_prompt( { "chat": { "system": [ @@ -213,9 +213,9 @@ def test_get_render_prompt_chat_dict_multiple_questions(mock_logger: Mock, mock_ # delete and import module to test if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt - result = _render_prompt( + result = _render_system_prompt( { "chat": { "messages": [ @@ -244,9 +244,9 @@ def test_get_render_prompt_chat_dict_multiple_assistant_prompts(mock_logger: Moc # delete and import module to test if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt - result = _render_prompt( + result = _render_system_prompt( { "chat": { "system": [ @@ -266,9 +266,9 @@ def test_get_render_prompt_chat_dict_multiple_assistant_message(mock_logger: Moc # delete and import module to test if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt - result = _render_prompt( + result = _render_system_prompt( { "chat": { "messages": [ @@ -297,9 +297,9 @@ def test_get_render_prompt_text_dict(mock_logger: Mock, mock_env: Mock): # delete and import module to test if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt - result = _render_prompt( + result = _render_system_prompt( { "text": "Second Prompt.", }, @@ -313,9 +313,9 @@ def test_get_render_prompt_empty(mock_logger: Mock, mock_env: Mock): # delete and import module to test if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt - result = _render_prompt( + result = _render_system_prompt( { "chat": { "system": [], @@ -332,11 +332,11 @@ def test_render_prompt_raises_configuration_error_empty(mock_logger): with patch("app.core.config.get_logger", return_value=mock_logger): if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt from app.services.exceptions import PromptLoadError with pytest.raises(PromptLoadError) as excinfo: - _render_prompt({}) + _render_system_prompt({}) # Verify the exception and logger call assert excinfo.type is PromptLoadError @@ -348,11 +348,11 @@ def test_render_prompt_raises_configuration_error_text_missing(mock_logger): with patch("app.core.config.get_logger", return_value=mock_logger): if "app.services.prompt_loader" in sys.modules: del sys.modules["app.services.prompt_loader"] - from app.services.prompt_loader import _render_prompt + from app.services.prompt_loader import _render_system_prompt from app.services.exceptions import PromptLoadError with pytest.raises(PromptLoadError) as excinfo: - _render_prompt({"text": {}}) + _render_system_prompt({"text": {}}) # Verify the exception and logger call assert excinfo.type is PromptLoadError diff --git a/packages/slackBotFunction/tests/test_query_reformulator.py b/packages/slackBotFunction/tests/test_query_reformulator.py deleted file mode 100644 index f9a629857..000000000 --- a/packages/slackBotFunction/tests/test_query_reformulator.py +++ /dev/null @@ -1,94 +0,0 @@ -import sys -import pytest -from unittest.mock import ANY, Mock, patch, MagicMock -from botocore.exceptions import ClientError - - -@pytest.fixture -def mock_logger(): - return MagicMock() - - -@patch("app.services.prompt_loader.load_prompt") -@patch("app.services.bedrock.invoke_model") -def test_reformulate_query_returns_string(mock_invoke_model: Mock, mock_load_prompt: Mock, mock_env: Mock): - """Test that reformulate_query returns a string without crashing""" - # set up mocks - mock_load_prompt.return_value = {"prompt_text": "Test reformat. {{user_query}}", "inference_config": {}} - mock_invoke_model.return_value = {"content": [{"text": "foo"}]} - - # delete and import module to test - if "app.services.query_reformulator" in sys.modules: - del sys.modules["app.services.query_reformulator"] - from app.services.query_reformulator import reformulate_query - - # perform operation - result = reformulate_query("How do I use EPS?") - - # assertions - # Function should return a string (either reformulated or fallback to original) - assert isinstance(result, str) - assert len(result) > 0 - assert result == "foo" - mock_load_prompt.assert_called_once_with("test-prompt", "DRAFT") - mock_invoke_model.assert_called_once_with( - prompt="Test reformat. How do I use EPS?", model_id="test-model", client=ANY, inference_config={} - ) - - -@patch("app.services.prompt_loader.load_prompt") -def test_reformulate_query_prompt_load_error(mock_load_prompt: Mock, mock_env: Mock): - # set up mocks - mock_load_prompt.side_effect = Exception("Prompt not found") - - # delete and import module to test - if "app.services.query_reformulator" in sys.modules: - del sys.modules["app.services.query_reformulator"] - from app.services.query_reformulator import reformulate_query - - # perform operation - original_query = "How do I use EPS?" - result = reformulate_query(original_query) - - # assertions - assert result == original_query - - -@patch("app.services.prompt_loader.load_prompt") -@patch("app.services.bedrock.invoke_model") -def test_reformulate_query_bedrock_error(mock_invoke_model: Mock, mock_load_prompt: Mock, mock_env: Mock): - """Test query reformulation with Bedrock API error""" - # set up mocks - mock_load_prompt.return_value = "Reformulate this query: {{user_query}}" - mock_invoke_model.side_effect = ClientError({"Error": {"Code": "ThrottlingException"}}, "InvokeModel") - - # delete and import module to test - if "app.services.query_reformulator" in sys.modules: - del sys.modules["app.services.query_reformulator"] - from app.services.query_reformulator import reformulate_query - - # perform operation - result = reformulate_query("original query") - - # assertions - assert result == "original query" - - -@patch("app.services.prompt_loader.load_prompt") -@patch("app.services.bedrock.invoke_model") -def test_reformulate_query_bedrock_invoke_model(mock_invoke_model: Mock, mock_load_prompt: Mock, mock_env: Mock): - """Test query reformulation with successful Bedrock invoke_model call""" - # set up mocks - mock_load_prompt.return_value = {"prompt_text": "Reformulate this query: {{user_query}}"} - mock_invoke_model.return_value = {"content": [{"text": "reformulated query"}]} - - # delete and import module to test - if "app.services.query_reformulator" in sys.modules: - del sys.modules["app.services.query_reformulator"] - from app.services.query_reformulator import reformulate_query - - # perform operation - result = reformulate_query("original query") - - # assertions - assert result == "reformulated query" From e36ca29276ddb42f9edee7c9c37f0fd1993d719d Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Fri, 13 Feb 2026 12:19:17 +0000 Subject: [PATCH 09/21] feat: Move prompt reformulation to rag orchestration --- packages/cdk/resources/BedrockPromptResources.ts | 12 ++++++------ packages/cdk/resources/BedrockPromptSettings.ts | 16 ++++++++-------- packages/cdk/resources/Functions.ts | 4 ++-- packages/cdk/resources/RuntimePolicies.ts | 4 ++-- packages/cdk/stacks/EpsAssistMeStack.ts | 16 +++++----------- .../app/services/ai_processor.py | 2 +- packages/slackBotFunction/tests/conftest.py | 4 ++-- .../slackBotFunction/tests/test_ai_processor.py | 2 +- 8 files changed, 27 insertions(+), 33 deletions(-) diff --git a/packages/cdk/resources/BedrockPromptResources.ts b/packages/cdk/resources/BedrockPromptResources.ts index 077c1f708..37ce1b09b 100644 --- a/packages/cdk/resources/BedrockPromptResources.ts +++ b/packages/cdk/resources/BedrockPromptResources.ts @@ -15,7 +15,7 @@ export interface BedrockPromptResourcesProps { } export class BedrockPromptResources extends Construct { - public readonly orchestrationReformulationPrompt: Prompt + public readonly orchestrationPrompt: Prompt public readonly ragResponsePrompt: Prompt public readonly modelId: string @@ -25,14 +25,14 @@ export class BedrockPromptResources extends Construct { const aiModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") // Create Prompts - this.orchestrationReformulationPrompt = this.createPrompt( - "OrchestrationReformulationPrompt", - `${props.stackName}-OrchestrationReformulation`, + this.orchestrationPrompt = this.createPrompt( + "OrchestrationPrompt", + `${props.stackName}-Orchestration`, "Prompt for orchestrating queries to improve RAG inference", aiModel, "", - [props.settings.reformulationPrompt], - props.settings.reformulationInferenceConfig + [props.settings.orchestrationPrompt], + props.settings.orchestrationInferenceConfig ) this.ragResponsePrompt = this.createPrompt( diff --git a/packages/cdk/resources/BedrockPromptSettings.ts b/packages/cdk/resources/BedrockPromptSettings.ts index 0b45f14ab..fda025243 100644 --- a/packages/cdk/resources/BedrockPromptSettings.ts +++ b/packages/cdk/resources/BedrockPromptSettings.ts @@ -3,18 +3,18 @@ import {ChatMessage} from "@cdklabs/generative-ai-cdk-constructs/lib/cdk-lib/bed import {Construct} from "constructs" import {CfnPrompt} from "aws-cdk-lib/aws-bedrock" -export type BedrockPromptSettingsType = "system" | "user" | "reformulation" +export type BedrockPromptSettingsType = "system" | "orchestration" | "user" /** BedrockPromptSettings is responsible for loading and providing - * the system, user, and reformulation prompts along with their + * the system, user, and orchestration prompts along with their * inference configurations. */ export class BedrockPromptSettings extends Construct { public readonly systemPrompt: ChatMessage public readonly userPrompt: ChatMessage - public readonly reformulationPrompt: ChatMessage + public readonly orchestrationPrompt: ChatMessage public readonly ragInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty - public readonly reformulationInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty + public readonly orchestrationInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty /** * @param scope The Construct scope @@ -30,8 +30,8 @@ export class BedrockPromptSettings extends Construct { const userPromptData = this.getTypedPrompt("user") this.userPrompt = ChatMessage.user(userPromptData.text) - const reformulationPrompt = this.getTypedPrompt("reformulation") - this.reformulationPrompt = ChatMessage.user(reformulationPrompt.text) + const orchestrationPrompt = this.getTypedPrompt("orchestration") + this.orchestrationPrompt = ChatMessage.user(orchestrationPrompt.text) this.ragInferenceConfig = { temperature: 0, @@ -42,7 +42,7 @@ export class BedrockPromptSettings extends Construct { ] } - this.reformulationInferenceConfig = { + this.orchestrationInferenceConfig = { temperature: 0.5, topP: 0.9, maxTokens: 512, @@ -56,7 +56,7 @@ export class BedrockPromptSettings extends Construct { * If a version is provided, it retrieves that specific version. * Otherwise, it retrieves the latest version based on file naming. * - * @param type The type of prompt (system, user, reformulation) + * @param type The type of prompt (system, user, orchestration) * @returns An object containing the prompt text and filename */ private getTypedPrompt(type: BedrockPromptSettingsType) diff --git a/packages/cdk/resources/Functions.ts b/packages/cdk/resources/Functions.ts index 50557f9bf..a648211d9 100644 --- a/packages/cdk/resources/Functions.ts +++ b/packages/cdk/resources/Functions.ts @@ -35,7 +35,7 @@ export interface FunctionsProps { readonly isPullRequest: boolean readonly mainSlackBotLambdaExecutionRoleArn : string readonly ragModelId: string - readonly queryReformulationModelId: string + readonly orchestrationModelId: string readonly notifyS3UploadFunctionPolicy: ManagedPolicy readonly docsBucketName: string } @@ -61,7 +61,7 @@ export class Functions extends Construct { dependencyLocation: ".dependencies/slackBotFunction", environmentVariables: { "RAG_MODEL_ID": props.ragModelId, - "QUERY_REFORMULATION_MODEL_ID": props.queryReformulationModelId, + "ORCHESTRATION_MODEL_ID": props.orchestrationModelId, "KNOWLEDGEBASE_ID": props.knowledgeBaseId, "LAMBDA_MEMORY_SIZE": LAMBDA_MEMORY_SIZE, "SLACK_BOT_TOKEN_PARAMETER": props.slackBotTokenParameter.parameterName, diff --git a/packages/cdk/resources/RuntimePolicies.ts b/packages/cdk/resources/RuntimePolicies.ts index df35703b6..dda752df7 100644 --- a/packages/cdk/resources/RuntimePolicies.ts +++ b/packages/cdk/resources/RuntimePolicies.ts @@ -13,7 +13,7 @@ export interface RuntimePoliciesProps { readonly dataSourceArn: string readonly promptName: string readonly ragModelId: string - readonly queryReformulationModelId: string + readonly orchestrationModelId: string readonly docsBucketArn: string readonly docsBucketKmsKeyArn: string } @@ -32,7 +32,7 @@ export class RuntimePolicies extends Construct { actions: ["bedrock:InvokeModel"], resources: [ `arn:aws:bedrock:${props.region}::foundation-model/${props.ragModelId}`, - `arn:aws:bedrock:${props.region}::foundation-model/${props.queryReformulationModelId}` + `arn:aws:bedrock:${props.region}::foundation-model/${props.orchestrationModelId}` ] }) diff --git a/packages/cdk/stacks/EpsAssistMeStack.ts b/packages/cdk/stacks/EpsAssistMeStack.ts index 62174ffc1..272a2cb97 100644 --- a/packages/cdk/stacks/EpsAssistMeStack.ts +++ b/packages/cdk/stacks/EpsAssistMeStack.ts @@ -163,9 +163,9 @@ export class EpsAssistMeStack extends Stack { knowledgeBaseArn: vectorKB.knowledgeBase.attrKnowledgeBaseArn, guardrailArn: vectorKB.guardrail.guardrailArn, dataSourceArn: vectorKB.dataSourceArn, - promptName: bedrockPromptResources.queryReformulationPrompt.promptName, + promptName: bedrockPromptResources.orchestrationPrompt.promptName, ragModelId: bedrockPromptResources.modelId, - queryReformulationModelId: bedrockPromptResources.modelId, + orchestrationModelId: bedrockPromptResources.modelId, docsBucketArn: storage.kbDocsBucket.bucketArn, docsBucketKmsKeyArn: storage.kbDocsKmsKey.keyArn }) @@ -192,12 +192,12 @@ export class EpsAssistMeStack extends Stack { slackBotTokenSecret: secrets.slackBotTokenSecret, slackBotSigningSecret: secrets.slackBotSigningSecret, slackBotStateTable: tables.slackBotStateTable.table, - orchestrationPromptName: bedrockPromptResources.queryReformulationPrompt.promptName, + orchestrationPromptName: bedrockPromptResources.orchestrationPrompt.promptName, ragResponsePromptName: bedrockPromptResources.ragResponsePrompt.promptName, - orchestrationPromptVersion: bedrockPromptResources.queryReformulationPrompt.promptVersion, + orchestrationPromptVersion: bedrockPromptResources.orchestrationPrompt.promptVersion, ragResponsePromptVersion: bedrockPromptResources.ragResponsePrompt.promptVersion, ragModelId: bedrockPromptResources.modelId, - queryReformulationModelId: bedrockPromptResources.modelId, + orchestrationModelId: bedrockPromptResources.modelId, isPullRequest: isPullRequest, mainSlackBotLambdaExecutionRoleArn: mainSlackBotLambdaExecutionRoleArn, notifyS3UploadFunctionPolicy: runtimePolicies.notifyS3UploadFunctionPolicy, @@ -279,12 +279,6 @@ export class EpsAssistMeStack extends Stack { description: "Slack Commands API endpoint for /slash commands" }) - // Output: Bedrock Prompt ARN - new CfnOutput(this, "QueryReformulationPromptArn", { - value: bedrockPromptResources.queryReformulationPrompt.promptArn, - description: "ARN of the query reformulation prompt in Bedrock" - }) - new CfnOutput(this, "kbDocsBucketArn", { value: storage.kbDocsBucket.bucketArn, exportName: `${props.stackName}:kbDocsBucket:Arn` diff --git a/packages/slackBotFunction/app/services/ai_processor.py b/packages/slackBotFunction/app/services/ai_processor.py index fab30e966..3cd0a058d 100644 --- a/packages/slackBotFunction/app/services/ai_processor.py +++ b/packages/slackBotFunction/app/services/ai_processor.py @@ -2,7 +2,7 @@ shared AI processing service - extracted to avoid duplication both slack handlers and direct invocation use identical logic for query -reformulation and bedrock interaction. single source of truth for AI flows. +orchestration and bedrock interaction. single source of truth for AI flows. """ from app.services.bedrock import query_bedrock diff --git a/packages/slackBotFunction/tests/conftest.py b/packages/slackBotFunction/tests/conftest.py index 7271b67f0..4264f051c 100644 --- a/packages/slackBotFunction/tests/conftest.py +++ b/packages/slackBotFunction/tests/conftest.py @@ -17,13 +17,13 @@ def mock_env(): "SLACK_SIGNING_SECRET_PARAMETER": "/test/signing-secret", "SLACK_BOT_STATE_TABLE": "test-bot-state-table", "KNOWLEDGEBASE_ID": "test-kb-id", - "RAG_MODEL_ID": "test-model-id", "AWS_REGION": "eu-west-2", "GUARD_RAIL_ID": "test-guard-id", "GUARD_RAIL_VERSION": "1", - "QUERY_REFORMULATION_MODEL_ID": "test-model", + "ORCHESTRATION_MODEL_ID": "test-model", "ORCHESTRATION_RESPONSE_PROMPT_NAME": "test-prompt", "ORCHESTRATION_RESPONSE_PROMPT_VERSION": "DRAFT", + "RAG_MODEL_ID": "test-model-id", "RAG_RESPONSE_PROMPT_NAME": "test-rag-prompt", "RAG_RESPONSE_PROMPT_VERSION": "DRAFT", } diff --git a/packages/slackBotFunction/tests/test_ai_processor.py b/packages/slackBotFunction/tests/test_ai_processor.py index 66249e89e..8fa89d682 100644 --- a/packages/slackBotFunction/tests/test_ai_processor.py +++ b/packages/slackBotFunction/tests/test_ai_processor.py @@ -1,4 +1,4 @@ -"""shared ai processor - validates query reformulation and bedrock integration""" +"""shared ai processor - validates query orchestration and bedrock integration""" import pytest from unittest.mock import patch From 416765cd347eb0e96075982183c2216d7151d643 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Fri, 13 Feb 2026 13:12:13 +0000 Subject: [PATCH 10/21] feat: Move prompt reformulation to rag orchestration --- .../slackBotFunction/app/services/bedrock.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/slackBotFunction/app/services/bedrock.py b/packages/slackBotFunction/app/services/bedrock.py index ec5305952..60019ea38 100644 --- a/packages/slackBotFunction/app/services/bedrock.py +++ b/packages/slackBotFunction/app/services/bedrock.py @@ -62,15 +62,15 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat } }, }, - }, - "orchestrationConfiguration": { - "inferenceConfig": { - "textInferenceConfig": { - **inference_config, - "stopSequences": [ - "Human:", - ], - } + "orchestrationConfiguration": { + "inferenceConfig": { + "textInferenceConfig": { + **inference_config, + "stopSequences": [ + "Human:", + ], + } + }, }, }, }, @@ -85,9 +85,9 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat ) if orchestration_prompt_template: - request_params["retrieveAndGenerateConfiguration"]["orchestrationConfiguration"]["promptTemplate"] = { - "textPromptTemplate": orchestration_prompt_template.get("prompt_text") - } + request_params["retrieveAndGenerateConfiguration"]["knowledgeBaseConfiguration"]["orchestrationConfiguration"][ + "promptTemplate" + ] = {"textPromptTemplate": orchestration_prompt_template.get("prompt_text")} logger.info( "Using prompt template for RAG response generation", extra={"prompt_name": config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME}, From fde2d878abc81f9037509b19d59e58f2019756cd Mon Sep 17 00:00:00 2001 From: Anthony Brown Date: Fri, 13 Feb 2026 14:56:14 +0000 Subject: [PATCH 11/21] fix sync file --- scripts/run_sync.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/run_sync.sh b/scripts/run_sync.sh index a3e1e6b12..aaff218f4 100755 --- a/scripts/run_sync.sh +++ b/scripts/run_sync.sh @@ -48,6 +48,8 @@ SLACK_SIGNING_SECRET=$(echo "$CF_LONDON_EXPORTS" | \ -r '.Exports[] | select(.Name == $EXPORT_NAME) | .Value') LOG_RETENTION_IN_DAYS=30 LOG_LEVEL=debug +FORWARD_CSOC_LOGS=false +RUN_REGRESSION_TESTS=false # export all the vars so they can be picked up by external programs export STACK_NAME @@ -57,22 +59,25 @@ export SLACK_BOT_TOKEN export SLACK_SIGNING_SECRET export LOG_RETENTION_IN_DAYS export LOG_LEVEL - +export FORWARD_CSOC_LOGS +export RUN_REGRESSION_TESTS echo "Generating config for ${EPSAM_CONFIG}" "$FIX_SCRIPT" "$EPSAM_CONFIG" echo "Installing dependencies locally" mkdir -p .dependencies -poetry export --without-hashes --format=requirements.txt --with slackBotFunction > .dependencies/requirements_slackBotFunction -poetry export --without-hashes --format=requirements.txt --with syncKnowledgeBaseFunction > .dependencies/requirements_syncKnowledgeBaseFunction -poetry export --without-hashes --format=requirements.txt --with preprocessingFunction > .dependencies/requirements_preprocessingFunction poetry show --only=slackBotFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > .dependencies/requirements_slackBotFunction poetry show --only=syncKnowledgeBaseFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > .dependencies/requirements_syncKnowledgeBaseFunction +poetry show --only=notifyS3UploadFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > .dependencies/requirements_notifyS3UploadFunction +poetry show --only=preprocessingFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > .dependencies/requirements_preprocessingFunction +poetry show --only=bedrockLoggingConfigFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > .dependencies/requirements_bedrockLoggingConfigFunction + pip3 install -r .dependencies/requirements_slackBotFunction -t .dependencies/slackBotFunction/python pip3 install -r .dependencies/requirements_syncKnowledgeBaseFunction -t .dependencies/syncKnowledgeBaseFunction/python pip3 install -r .dependencies/requirements_notifyS3UploadFunction -t .dependencies/notifyS3UploadFunction/python pip3 install -r .dependencies/requirements_preprocessingFunction -t .dependencies/preprocessingFunction/python +pip3 install -r .dependencies/requirements_bedrockLoggingConfigFunction -t .dependencies/bedrockLoggingConfigFunction/python rm -rf .dependencies/preprocessingFunction/python/magika* .dependencies/preprocessingFunction/python/onnxruntime* cp packages/preprocessingFunction/magika_shim.py .dependencies/preprocessingFunction/python/magika.py find .dependencies/preprocessingFunction/python -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true From 2d502b07bd327f1c8731a72a2b0ccb0876c5dd90 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Fri, 13 Feb 2026 17:36:29 +0000 Subject: [PATCH 12/21] feat: Use orchistration prompt to improve user prompt --- packages/cdk/prompts/orchestrationPrompt.txt | 60 +++++++++---------- packages/cdk/prompts/systemPrompt.txt | 8 +-- packages/cdk/prompts/userPrompt.txt | 2 + .../cdk/resources/BedrockPromptSettings.ts | 2 +- .../app/services/ai_processor.py | 17 +++++- .../slackBotFunction/app/services/bedrock.py | 43 ++++--------- scripts/run_sync.sh | 3 + 7 files changed, 63 insertions(+), 72 deletions(-) diff --git a/packages/cdk/prompts/orchestrationPrompt.txt b/packages/cdk/prompts/orchestrationPrompt.txt index acecd43ef..fad249bd3 100644 --- a/packages/cdk/prompts/orchestrationPrompt.txt +++ b/packages/cdk/prompts/orchestrationPrompt.txt @@ -1,37 +1,33 @@ -You are a query creation agent. You will be provided with a function and a description of what it searches over. The user will provide you a question, and your job is to determine the optimal query to use based on the user's question. -Here are a few examples of queries formed by other search function selection and query creation agents: +# Query Optimization Task - - - What if my vehicle is totaled in an accident? - what happens if my vehicle is totaled - - - I am relocating within the same state. Can I keep my current agent? - can I keep my current agent when moving in state - - - -You should also pay attention to the conversation history between the user and the search engine in order to gain the context necessary to create the query. -Here's another example that shows how you should reference the conversation history when generating a query: +## Instructions +Analyze the user query, detect all questions and sub-questions, and optimize each one for better search results. Use the search results provided to enhance the query with relevant terminology, context, and a better structure for search algorithms. - - - - How many vehicles can I include in a quote in Kansas - You can include 5 vehicles in a quote if you live in Kansas - - - What about texas? - You can include 3 vehicles in a quote if you live in Texas - - - + +$search_results$ + -IMPORTANT: the elements in the tags should not be assumed to have been provided to you to use UNLESS they are also explicitly given to you below. -All of the values and information within the examples (the questions, answers, and function calls) are strictly part of the examples and have not been provided to you. + +{{user_query}} + -Here is the current conversation history: -$conversation_history$ +## Examples for Reference + +### Example 1 +#### Input +I have a ferrari and my key battery has run out, is it possible that I can unlock it any other way?" ++ "Can I use the key physically?" ++ "Is there a number I can ring?" -$output_format_instructions$ +#### Output + +* Provide all possible ways of unlocking a farrari besides using a wireless key. +1. Inform physical key possibilities. +2. Provide Farrarri contact number. + + + +## Constraints (CRITICAL) +- You must wrap your final optimized query inside and tags. +- DO NOT output any headings, introductions, or conversational text outside of these tags. +- DO NOT wrap the actual query in quotation marks. diff --git a/packages/cdk/prompts/systemPrompt.txt b/packages/cdk/prompts/systemPrompt.txt index d29b9b3bc..4a7ec2f50 100644 --- a/packages/cdk/prompts/systemPrompt.txt +++ b/packages/cdk/prompts/systemPrompt.txt @@ -6,7 +6,7 @@ STYLE & FORMATTING RULES: - Do NOT add a "Citations" section at the end of the response. - Do NOT reference how the information was found (e.g., "...the provided search results") - Text should prioritie readability. -- Links should use Markdown text, e.g., [link text](url). +- Links should use Markdown text, e.g., . - Use `Inline Code` for system names, field names, or technical terms (e.g., `HL7 FHIR`). STEPS: @@ -17,12 +17,10 @@ STEPS: EXAMPLE: *Summary* -This is a short answer the captures the core question provided. +This is a short, fast answer so the user doesn't _have_ to read the long answer. *Answer* -This is a direct answer to the question, or questions, provided. It is in-depth, and breaks down individual questions. There is no reference to the text here (for example, you don't see "from source 1") but instead treats this information as if it was public knowledge. However, if there is a source, it does provide that source [as a hyperlink](hyperlink) to the website it can be found. +This is a direct answer to the question, or questions, provided. It breaks down individual questions. There is no reference to the text here (for example, you don't see "from source 1") but instead treats this information as if it was public knowledge. However, if there is a source, it does provide that source [as a hyperlink](hyperlink) to the website it can be found. There is multiple paragraphs, with blank lines between, to make it easier to read, as readability is a requirement. - -For more details, please refer to the [Authentication Guide](http://example.com/guide). diff --git a/packages/cdk/prompts/userPrompt.txt b/packages/cdk/prompts/userPrompt.txt index e54881843..50ed2db93 100644 --- a/packages/cdk/prompts/userPrompt.txt +++ b/packages/cdk/prompts/userPrompt.txt @@ -2,3 +2,5 @@ $search_results$ {{user_query}} + +Answers: diff --git a/packages/cdk/resources/BedrockPromptSettings.ts b/packages/cdk/resources/BedrockPromptSettings.ts index fda025243..0fad4a5af 100644 --- a/packages/cdk/resources/BedrockPromptSettings.ts +++ b/packages/cdk/resources/BedrockPromptSettings.ts @@ -31,7 +31,7 @@ export class BedrockPromptSettings extends Construct { this.userPrompt = ChatMessage.user(userPromptData.text) const orchestrationPrompt = this.getTypedPrompt("orchestration") - this.orchestrationPrompt = ChatMessage.user(orchestrationPrompt.text) + this.orchestrationPrompt = ChatMessage.assistant(orchestrationPrompt.text) this.ragInferenceConfig = { temperature: 0, diff --git a/packages/slackBotFunction/app/services/ai_processor.py b/packages/slackBotFunction/app/services/ai_processor.py index 3cd0a058d..0e8478bfd 100644 --- a/packages/slackBotFunction/app/services/ai_processor.py +++ b/packages/slackBotFunction/app/services/ai_processor.py @@ -6,8 +6,9 @@ """ from app.services.bedrock import query_bedrock -from app.core.config import get_logger +from app.core.config import get_retrieve_generate_config, get_logger from app.core.types import AIProcessorResponse +from app.services.prompt_loader import load_prompt logger = get_logger() @@ -15,7 +16,19 @@ def process_ai_query(user_query: str, session_id: str | None = None) -> AIProcessorResponse: """shared AI processing logic for both slack and direct invocation""" # session_id enables conversation continuity across multiple queries - kb_response = query_bedrock(user_query, session_id) + config = get_retrieve_generate_config() + + orchestration_prompt_template = load_prompt( + config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME, + config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_VERSION, + ) + orchestrated_prompt = query_bedrock(user_query, orchestration_prompt_template, config, session_id) + orchestrated_text = orchestrated_prompt["output"]["text"] + + logger.debug("Orchestrated_text", extra={"text": orchestrated_text}) + + rag_prompt_template = load_prompt(config.RAG_RESPONSE_PROMPT_NAME, config.RAG_RESPONSE_PROMPT_VERSION) + kb_response = query_bedrock(orchestrated_text, rag_prompt_template, config, session_id) logger.info( "response from bedrock", diff --git a/packages/slackBotFunction/app/services/bedrock.py b/packages/slackBotFunction/app/services/bedrock.py index 60019ea38..f4e5e98d5 100644 --- a/packages/slackBotFunction/app/services/bedrock.py +++ b/packages/slackBotFunction/app/services/bedrock.py @@ -5,14 +5,18 @@ from mypy_boto3_bedrock_runtime.client import BedrockRuntimeClient from mypy_boto3_bedrock_agent_runtime.type_defs import RetrieveAndGenerateResponseTypeDef -from app.core.config import get_retrieve_generate_config, get_logger -from app.services.prompt_loader import load_prompt +from app.core.config import get_logger logger = get_logger() -def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerateResponseTypeDef: +def query_bedrock( + user_query: str, + prompt_template: dict, + config: BedrockConfig, + session_id: str = None, +) -> RetrieveAndGenerateResponseTypeDef: """ Query Amazon Bedrock Knowledge Base using RAG (Retrieval-Augmented Generation) @@ -20,13 +24,7 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat a response using the configured LLM model with guardrails for safety. """ - config = get_retrieve_generate_config() - rag_prompt_template = load_prompt(config.RAG_RESPONSE_PROMPT_NAME, config.RAG_RESPONSE_PROMPT_VERSION) - orchestration_prompt_template = load_prompt( - config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME, - config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_VERSION, - ) - inference_config = rag_prompt_template.get("inference_config") + inference_config = prompt_template.get("inference_config") if not inference_config: default_values = {"temperature": 0, "maxTokens": 1024, "topP": 0.1} @@ -46,7 +44,7 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat "type": "KNOWLEDGE_BASE", "knowledgeBaseConfiguration": { "knowledgeBaseId": config.KNOWLEDGEBASE_ID, - "modelArn": rag_prompt_template.get("model_id", config.RAG_MODEL_ID), + "modelArn": prompt_template.get("model_id", config.RAG_MODEL_ID), "retrievalConfiguration": {"vectorSearchConfiguration": {"numberOfResults": 5}}, "generationConfiguration": { "guardrailConfiguration": { @@ -62,37 +60,18 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat } }, }, - "orchestrationConfiguration": { - "inferenceConfig": { - "textInferenceConfig": { - **inference_config, - "stopSequences": [ - "Human:", - ], - } - }, - }, }, }, } - if rag_prompt_template: + if prompt_template: request_params["retrieveAndGenerateConfiguration"]["knowledgeBaseConfiguration"]["generationConfiguration"][ "promptTemplate" - ] = {"textPromptTemplate": rag_prompt_template.get("prompt_text")} + ] = {"textPromptTemplate": prompt_template.get("prompt_text")} logger.info( "Using prompt template for RAG response generation", extra={"prompt_name": config.RAG_RESPONSE_PROMPT_NAME} ) - if orchestration_prompt_template: - request_params["retrieveAndGenerateConfiguration"]["knowledgeBaseConfiguration"]["orchestrationConfiguration"][ - "promptTemplate" - ] = {"textPromptTemplate": orchestration_prompt_template.get("prompt_text")} - logger.info( - "Using prompt template for RAG response generation", - extra={"prompt_name": config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME}, - ) - # Include session ID for conversation continuity across messages if session_id: request_params["sessionId"] = session_id diff --git a/scripts/run_sync.sh b/scripts/run_sync.sh index aaff218f4..c47bab717 100755 --- a/scripts/run_sync.sh +++ b/scripts/run_sync.sh @@ -78,8 +78,11 @@ pip3 install -r .dependencies/requirements_syncKnowledgeBaseFunction -t .depende pip3 install -r .dependencies/requirements_notifyS3UploadFunction -t .dependencies/notifyS3UploadFunction/python pip3 install -r .dependencies/requirements_preprocessingFunction -t .dependencies/preprocessingFunction/python pip3 install -r .dependencies/requirements_bedrockLoggingConfigFunction -t .dependencies/bedrockLoggingConfigFunction/python + rm -rf .dependencies/preprocessingFunction/python/magika* .dependencies/preprocessingFunction/python/onnxruntime* + cp packages/preprocessingFunction/magika_shim.py .dependencies/preprocessingFunction/python/magika.py + find .dependencies/preprocessingFunction/python -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true find .dependencies/preprocessingFunction/python -type d -name "test" -exec rm -rf {} + 2>/dev/null || true find .dependencies/preprocessingFunction/python -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true From 902032933e14e91b818b2fb4f5616ff4121e9157 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Mon, 16 Feb 2026 11:43:56 +0000 Subject: [PATCH 13/21] feat: fix unit tests --- packages/slackBotFunction/app/core/config.py | 4 +- .../app/services/ai_processor.py | 4 +- .../slackBotFunction/app/services/bedrock.py | 2 +- .../tests/test_ai_processor.py | 90 +++++++++++++++---- .../tests/test_bedrock_integration.py | 44 ++++----- scripts/run_sync.sh | 2 + 6 files changed, 96 insertions(+), 50 deletions(-) diff --git a/packages/slackBotFunction/app/core/config.py b/packages/slackBotFunction/app/core/config.py index 146a0f5d3..6cd591495 100644 --- a/packages/slackBotFunction/app/core/config.py +++ b/packages/slackBotFunction/app/core/config.py @@ -152,8 +152,8 @@ class BedrockConfig: GUARD_VERSION: str RAG_RESPONSE_PROMPT_NAME: str RAG_RESPONSE_PROMPT_VERSION: str - ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME: str - ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_VERSION: str + ORCHESTRATION_PROMPT_NAME: str + ORCHESTRATION_PROMPT_VERSION: str @dataclass diff --git a/packages/slackBotFunction/app/services/ai_processor.py b/packages/slackBotFunction/app/services/ai_processor.py index 0e8478bfd..1b5cb4355 100644 --- a/packages/slackBotFunction/app/services/ai_processor.py +++ b/packages/slackBotFunction/app/services/ai_processor.py @@ -19,8 +19,8 @@ def process_ai_query(user_query: str, session_id: str | None = None) -> AIProces config = get_retrieve_generate_config() orchestration_prompt_template = load_prompt( - config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_NAME, - config.ORCHESTRATION_RESPONSE_PROMPT_NAME_RESPONSE_PROMPT_VERSION, + config.ORCHESTRATION_PROMPT_NAME, + config.ORCHESTRATION_PROMPT_VERSION, ) orchestrated_prompt = query_bedrock(user_query, orchestration_prompt_template, config, session_id) orchestrated_text = orchestrated_prompt["output"]["text"] diff --git a/packages/slackBotFunction/app/services/bedrock.py b/packages/slackBotFunction/app/services/bedrock.py index f4e5e98d5..3cda7da7d 100644 --- a/packages/slackBotFunction/app/services/bedrock.py +++ b/packages/slackBotFunction/app/services/bedrock.py @@ -5,7 +5,7 @@ from mypy_boto3_bedrock_runtime.client import BedrockRuntimeClient from mypy_boto3_bedrock_agent_runtime.type_defs import RetrieveAndGenerateResponseTypeDef -from app.core.config import get_logger +from app.core.config import BedrockConfig, get_logger logger = get_logger() diff --git a/packages/slackBotFunction/tests/test_ai_processor.py b/packages/slackBotFunction/tests/test_ai_processor.py index 8fa89d682..7ded9be88 100644 --- a/packages/slackBotFunction/tests/test_ai_processor.py +++ b/packages/slackBotFunction/tests/test_ai_processor.py @@ -1,14 +1,29 @@ """shared ai processor - validates query orchestration and bedrock integration""" import pytest -from unittest.mock import patch +from unittest.mock import call, patch, ANY from app.services.ai_processor import process_ai_query +@pytest.fixture +def mock_config_setup(mock_load_prompt, mock_config): + """Setup common mock configurations""" + mock_load_prompt.return_value = {"prompt_text": "test_prompt", "model_id": "model_id", "inference_config": {}} + mock_config.get_retrieve_generate_config.return_value = { + "ORCHESTRATION_PROMPT_NAME": "test", + "ORCHESTRATION_PROMPT_VERSION": "test", + "RAG_RESPONSE_PROMPT_NAME": "test", + "RAG_RESPONSE_PROMPT_VERSION": "test", + } + return mock_load_prompt, mock_config + + class TestAIProcessor: + @patch("app.services.ai_processor.get_retrieve_generate_config") + @patch("app.services.ai_processor.load_prompt") @patch("app.services.ai_processor.query_bedrock") - def test_process_ai_query_without_session(self, mock_bedrock): + def test_process_ai_query_without_session(self, mock_bedrock, mock_load_prompt, mock_config): """new conversation: no session context passed to bedrock""" mock_bedrock.return_value = { "output": {"text": "To authenticate with EPS API, you need..."}, @@ -24,38 +39,68 @@ def test_process_ai_query_without_session(self, mock_bedrock): assert result["citations"][0]["title"] == "EPS Authentication Guide" assert "kb_response" in result - mock_bedrock.assert_called_once_with("How to authenticate EPS API?", None) - + assert mock_bedrock.call_count == 2 + assert mock_load_prompt.call_count == 2 + + mock_bedrock.assert_has_calls( + [ + call("How to authenticate EPS API?", mock_load_prompt.return_value, ANY, None), + call( + "To authenticate with EPS API, you need...", + mock_load_prompt.return_value, + ANY, + None, + ), + ] + ) + + @patch("app.services.ai_processor.get_retrieve_generate_config") + @patch("app.services.ai_processor.load_prompt") @patch("app.services.ai_processor.query_bedrock") - def test_process_ai_query_with_session(self, mock_bedrock): + def test_process_ai_query_with_session(self, mock_bedrock, mock_load_prompt, mock_config): """conversation continuity: existing session maintained across queries""" + mock_prompt = "What about rate limits?" + mock_session_id = "existing-session-456" mock_bedrock.return_value = { "output": {"text": "EPS API has rate limits of..."}, - "sessionId": "existing-session-456", + "sessionId": mock_session_id, "citations": [], } - result = process_ai_query("What about rate limits?", session_id="existing-session-456") + result = process_ai_query(mock_prompt, session_id="existing-session-456") assert result["text"] == "EPS API has rate limits of..." assert result["session_id"] == "existing-session-456" assert result["citations"] == [] assert "kb_response" in result - mock_bedrock.assert_called_once_with("What about rate limits?", "existing-session-456") - + mock_bedrock.assert_has_calls( + [ + call("What about rate limits?", mock_load_prompt.return_value, ANY, mock_session_id), + call( + "EPS API has rate limits of...", + mock_load_prompt.return_value, + ANY, + mock_session_id, + ), + ] + ) + + @patch("app.services.ai_processor.get_retrieve_generate_config") + @patch("app.services.ai_processor.load_prompt") @patch("app.services.ai_processor.query_bedrock") - def test_process_ai_query_bedrock_error(self, mock_bedrock): + def test_process_ai_query_bedrock_error(self, mock_bedrock, mock_load_prompt, mock_config): """bedrock service failure: error propagated to caller""" mock_bedrock.side_effect = Exception("Bedrock service error") - with pytest.raises(Exception) as exc_info: process_ai_query("How to authenticate EPS API?") assert "Bedrock service error" in str(exc_info.value) + @patch("app.services.ai_processor.get_retrieve_generate_config") + @patch("app.services.ai_processor.load_prompt") @patch("app.services.ai_processor.query_bedrock") - def test_process_ai_query_missing_citations(self, mock_bedrock): + def test_process_ai_query_missing_citations(self, mock_bedrock, mock_load_prompt, mock_config): """bedrock response incomplete: citations default to empty list""" mock_bedrock.return_value = { "output": {"text": "Response without citations"}, @@ -69,8 +114,10 @@ def test_process_ai_query_missing_citations(self, mock_bedrock): assert result["session_id"] == "session-123" assert result["citations"] == [] # safe default when bedrock omits citations + @patch("app.services.ai_processor.get_retrieve_generate_config") + @patch("app.services.ai_processor.load_prompt") @patch("app.services.ai_processor.query_bedrock") - def test_process_ai_query_missing_session_id(self, mock_bedrock): + def test_process_ai_query_missing_session_id(self, mock_bedrock, mock_load_prompt, mock_config): """bedrock response incomplete: session_id properly handles None""" mock_bedrock.return_value = { "output": {"text": "Response without session"}, @@ -84,8 +131,10 @@ def test_process_ai_query_missing_session_id(self, mock_bedrock): assert result["session_id"] is None # explicit None when bedrock omits sessionId assert result["citations"] == [] + @patch("app.services.ai_processor.get_retrieve_generate_config") + @patch("app.services.ai_processor.load_prompt") @patch("app.services.ai_processor.query_bedrock") - def test_process_ai_query_empty_query(self, mock_bedrock): + def test_process_ai_query_empty_query(self, mock_bedrock, mock_load_prompt, mock_config): """edge case: empty query still processed through full pipeline""" mock_bedrock.return_value = { "output": {"text": "Please provide a question"}, @@ -96,10 +145,19 @@ def test_process_ai_query_empty_query(self, mock_bedrock): result = process_ai_query("") assert result["text"] == "Please provide a question" - mock_bedrock.assert_called_once_with("", None) + mock_bedrock.assert_called_with + + mock_bedrock.assert_has_calls( + [ + call("", ANY, ANY, None), + call("Please provide a question", ANY, ANY, None), + ] + ) + @patch("app.services.ai_processor.get_retrieve_generate_config") + @patch("app.services.ai_processor.load_prompt") @patch("app.services.ai_processor.query_bedrock") - def test_process_ai_query_includes_raw_response(self, mock_bedrock): + def test_process_ai_query_includes_raw_response(self, mock_bedrock, mock_load_prompt, mock_config): """slack needs raw bedrock data: kb_response preserved for session handling""" raw_response = { "output": {"text": "Test response"}, diff --git a/packages/slackBotFunction/tests/test_bedrock_integration.py b/packages/slackBotFunction/tests/test_bedrock_integration.py index a7f752de5..75a72435b 100644 --- a/packages/slackBotFunction/tests/test_bedrock_integration.py +++ b/packages/slackBotFunction/tests/test_bedrock_integration.py @@ -1,10 +1,9 @@ import sys -from unittest.mock import Mock, patch +from unittest.mock import Mock, patch, MagicMock, ANY -@patch("app.services.prompt_loader.load_prompt") @patch("boto3.client") -def test_get_bedrock_knowledgebase_response(mock_boto_client: Mock, mock_load_prompt: Mock, mock_env: Mock): +def test_get_bedrock_knowledgebase_response(mock_boto_client: Mock, mock_env: Mock): """Test Bedrock knowledge base integration""" # set up mocks mock_client = Mock() @@ -17,18 +16,16 @@ def test_get_bedrock_knowledgebase_response(mock_boto_client: Mock, mock_load_pr from app.services.bedrock import query_bedrock # perform operation - result = query_bedrock("test query") + result = query_bedrock("test query", {"inference_config": None}, MagicMock()) # assertions - mock_load_prompt.assert_called_with("test-prompt", "DRAFT") - mock_boto_client.assert_called_once_with(service_name="bedrock-agent-runtime", region_name="eu-west-2") + mock_boto_client.assert_called_once_with(service_name="bedrock-agent-runtime", region_name=ANY) mock_client.retrieve_and_generate.assert_called_once() assert result["output"]["text"] == "bedrock response" -@patch("app.services.prompt_loader.load_prompt") @patch("boto3.client") -def test_query_bedrock_with_session(mock_boto_client: Mock, mock_load_prompt: Mock, mock_env: Mock): +def test_query_bedrock_with_session(mock_boto_client: Mock, mock_env: Mock): """Test query_bedrock with existing session""" # set up mocks mock_client = Mock() @@ -42,18 +39,16 @@ def test_query_bedrock_with_session(mock_boto_client: Mock, mock_load_prompt: Mo from app.services.bedrock import query_bedrock # perform operation - result = query_bedrock("test query", session_id="existing_session") + result = query_bedrock("test query", {"inference_config": None}, MagicMock(), session_id="existing_session") # assertions - mock_load_prompt.assert_called_with("test-prompt", "DRAFT") assert result == mock_response call_args = mock_client.retrieve_and_generate.call_args[1] assert call_args["sessionId"] == "existing_session" -@patch("app.services.prompt_loader.load_prompt") @patch("boto3.client") -def test_query_bedrock_without_session(mock_boto_client: Mock, mock_load_prompt: Mock, mock_env: Mock): +def test_query_bedrock_without_session(mock_boto_client: Mock, mock_env: Mock): """Test query_bedrock without session""" # set up mocks mock_client = Mock() @@ -67,24 +62,21 @@ def test_query_bedrock_without_session(mock_boto_client: Mock, mock_load_prompt: from app.services.bedrock import query_bedrock # perform operation - result = query_bedrock("test query") + result = query_bedrock("test query", {"inference_config": None}, MagicMock()) # assertions - mock_load_prompt.assert_called_with("test-prompt", "DRAFT") assert result == mock_response call_args = mock_client.retrieve_and_generate.call_args[1] assert "sessionId" not in call_args -@patch("app.services.prompt_loader.load_prompt") @patch("boto3.client") -def test_query_bedrock_check_prompt(mock_boto_client: Mock, mock_load_prompt: Mock, mock_env: Mock): +def test_query_bedrock_check_prompt(mock_boto_client: Mock, mock_env: Mock): """Test query_bedrock prompt loading""" # set up mocks mock_client = Mock() mock_boto_client.return_value = mock_client mock_client.retrieve_and_generate.return_value = {"output": {"text": "response"}} - mock_load_prompt.return_value = {"prompt_text": "Test prompt template", "inference_config": {}} # delete and import module to test if "app.services.bedrock" in sys.modules: @@ -92,10 +84,9 @@ def test_query_bedrock_check_prompt(mock_boto_client: Mock, mock_load_prompt: Mo from app.services.bedrock import query_bedrock # perform operation - result = query_bedrock("test query") + result = query_bedrock("test query", {"inference_config": None, "prompt_text": "Test prompt template"}, MagicMock()) # assertions - mock_load_prompt.assert_called_with("test-prompt", "DRAFT") call_args = mock_client.retrieve_and_generate.call_args[1] prompt_template = call_args["retrieveAndGenerateConfiguration"]["knowledgeBaseConfiguration"][ "generationConfiguration" @@ -104,18 +95,13 @@ def test_query_bedrock_check_prompt(mock_boto_client: Mock, mock_load_prompt: Mo assert result["output"]["text"] == "response" -@patch("app.services.prompt_loader.load_prompt") @patch("boto3.client") -def test_query_bedrock_check_config(mock_boto_client: Mock, mock_load_prompt: Mock, mock_env: Mock): +def test_query_bedrock_check_config(mock_boto_client: Mock, mock_env: Mock): """Test query_bedrock config loading""" # set up mocks mock_client = Mock() mock_boto_client.return_value = mock_client mock_client.retrieve_and_generate.return_value = {"output": {"text": "response"}} - mock_load_prompt.return_value = { - "prompt_text": "Test prompt template", - "inference_config": {"temperature": "0", "maxTokens": "1024", "topP": "0.1"}, - } # delete and import module to test if "app.services.bedrock" in sys.modules: @@ -123,7 +109,7 @@ def test_query_bedrock_check_config(mock_boto_client: Mock, mock_load_prompt: Mo from app.services.bedrock import query_bedrock # perform operation - query_bedrock("test query") + query_bedrock("test query", {"inference_config": None}, MagicMock()) # assertions call_args = mock_client.retrieve_and_generate.call_args[1] @@ -131,6 +117,6 @@ def test_query_bedrock_check_config(mock_boto_client: Mock, mock_load_prompt: Mo "generationConfiguration" ]["inferenceConfig"]["textInferenceConfig"] - assert prompt_config["temperature"] == "0" - assert prompt_config["maxTokens"] == "1024" - assert prompt_config["topP"] == "0.1" + assert prompt_config["temperature"] == 0 + assert prompt_config["maxTokens"] == 1024 + assert prompt_config["topP"] == 0.1 diff --git a/scripts/run_sync.sh b/scripts/run_sync.sh index c47bab717..a0dc4b136 100755 --- a/scripts/run_sync.sh +++ b/scripts/run_sync.sh @@ -51,6 +51,8 @@ LOG_LEVEL=debug FORWARD_CSOC_LOGS=false RUN_REGRESSION_TESTS=false + + # export all the vars so they can be picked up by external programs export STACK_NAME export COMMIT_ID From 2ca9b2b3bf2921afaa779bde6a0af230c537a8e8 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Mon, 16 Feb 2026 12:27:26 +0000 Subject: [PATCH 14/21] feat: Improve refinement prompt --- packages/cdk/prompts/orchestrationPrompt.txt | 45 ++++++++++--------- .../cdk/resources/BedrockPromptSettings.ts | 4 +- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/packages/cdk/prompts/orchestrationPrompt.txt b/packages/cdk/prompts/orchestrationPrompt.txt index fad249bd3..4a7de402f 100644 --- a/packages/cdk/prompts/orchestrationPrompt.txt +++ b/packages/cdk/prompts/orchestrationPrompt.txt @@ -3,31 +3,36 @@ ## Instructions Analyze the user query, detect all questions and sub-questions, and optimize each one for better search results. Use the search results provided to enhance the query with relevant terminology, context, and a better structure for search algorithms. - +### Search Context $search_results$ - - +### User Query {{user_query}} - -## Examples for Reference - -### Example 1 -#### Input -I have a ferrari and my key battery has run out, is it possible that I can unlock it any other way?" -+ "Can I use the key physically?" -+ "Is there a number I can ring?" +## Task +Your task is to: +1. Identify all questions and sub-questions in the user query +2. Restructure them in a clear, organized format +3. Enhance each question with relevant terminology from the search results +4. Create a comprehensive optimized query that will yield better search results -#### Output +## Output Format +Provide your optimized query directly without any preamble, explanations, or additional text. + +## Constraints +- Your response must be wrapped inside and tags +- DO NOT include any headings, introductions, or conversational text outside these tags +- DO NOT wrap the actual query in quotation marks +- Include all important information, examples, and constraints from the original query +- Keep questions comprehensive, prioritising easy understanding for end users + +## Example +Input: "I have a ferrari and my key battery has run out, is it possible that I can unlock it any other way? Can I use the key physically? Is there a number I can ring? My reg is 123456, is this important" + +Output: -* Provide all possible ways of unlocking a farrari besides using a wireless key. +* Provide all possible ways of unlocking a Ferrari besides using a wireless key. 1. Inform physical key possibilities. -2. Provide Farrarri contact number. +2. Provide Ferrari contact number. +3. Is proving the registration important (reg number - 123456) - - -## Constraints (CRITICAL) -- You must wrap your final optimized query inside and tags. -- DO NOT output any headings, introductions, or conversational text outside of these tags. -- DO NOT wrap the actual query in quotation marks. diff --git a/packages/cdk/resources/BedrockPromptSettings.ts b/packages/cdk/resources/BedrockPromptSettings.ts index 0fad4a5af..c99bf7a76 100644 --- a/packages/cdk/resources/BedrockPromptSettings.ts +++ b/packages/cdk/resources/BedrockPromptSettings.ts @@ -43,8 +43,8 @@ export class BedrockPromptSettings extends Construct { } this.orchestrationInferenceConfig = { - temperature: 0.5, - topP: 0.9, + temperature: 0, + topP: 0.3, maxTokens: 512, stopSequences: [ "Human:" From 7f56957edde74c913f4792e86a8971bea70d5b2d Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Mon, 16 Feb 2026 16:11:15 +0000 Subject: [PATCH 15/21] feat: No sessionId for orchestration --- packages/cdk/prompts/orchestrationPrompt.txt | 68 +++++++++++-------- packages/cdk/prompts/systemPrompt.txt | 10 +-- .../app/services/ai_processor.py | 2 +- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/packages/cdk/prompts/orchestrationPrompt.txt b/packages/cdk/prompts/orchestrationPrompt.txt index 4a7de402f..615df133f 100644 --- a/packages/cdk/prompts/orchestrationPrompt.txt +++ b/packages/cdk/prompts/orchestrationPrompt.txt @@ -1,38 +1,46 @@ -# Query Optimization Task +<|begin_of_text|><|start_header_id|>system<|end_header_id|> +You are an expert RAG query and context optimizer. Your task is to analyze verbose user queries and raw search context, stripping away all conversational filler to output a concise, impactful summary. -## Instructions -Analyze the user query, detect all questions and sub-questions, and optimize each one for better search results. Use the search results provided to enhance the query with relevant terminology, context, and a better structure for search algorithms. +You must: +1. Extract the core objective into a single, direct question. +2. Capture individual questions and their specific needs. +3. Isolate critical variables, specific states, and constraints required to solve the problem. +4. Enhance the question(s) with relevant terminology from the search results +Output your response strictly using the following XML structure: + (The short, direct question) + (Bullet points of critical states, statuses, or constraints) +<|eot_id|> + +<|start_header_id|>user<|end_header_id|> +### Search Context +The company leave policy dictates that standard PTO accrues at 1.5 days per month. However, employees on probation (first 90 days) cannot use accrued PTO. If an employee transitions from part-time to full-time, their prior accrued PTO carries over, but the new accrual rate of 1.5 days begins on the official transition date. Negative PTO balances are only permitted for full-time employees with at least 1 year of tenure, up to a maximum of -3 days. + +### User Query +Hi, I need some help figuring out the PTO rules for one of my team members. They started as part-time 6 months ago, but they just transitioned to full-time last week (let's say exactly 7 days ago). They currently have 2 days of PTO saved up from their part-time stint. They want to take next week off entirely, which would require 5 days of PTO. Can they do this, effectively going to a -3 balance, since they are full-time now? +<|eot_id|> + +<|start_header_id|>assistant<|end_header_id|> + +Can a recently transitioned full-time employee with 6 months total tenure and 2 accrued PTO days take 5 days off, resulting in a -3 PTO balance? + + +- Current Status: Full-time (transitioned 7 days ago) +- Total Tenure: 6 months +- Current PTO Balance: 2 days +- Requested PTO: 5 days (resulting in -3 balance) + + +Part-time PTO carries over upon transitioning to full-time. Negative PTO balances (up to -3 days) are strictly reserved for full-time employees with at least 1 year of tenure. + +<|eot_id|> + +<|start_header_id|>user<|end_header_id|> ### Search Context $search_results$ ### User Query {{user_query}} +<|eot_id|> -## Task -Your task is to: -1. Identify all questions and sub-questions in the user query -2. Restructure them in a clear, organized format -3. Enhance each question with relevant terminology from the search results -4. Create a comprehensive optimized query that will yield better search results - -## Output Format -Provide your optimized query directly without any preamble, explanations, or additional text. - -## Constraints -- Your response must be wrapped inside and tags -- DO NOT include any headings, introductions, or conversational text outside these tags -- DO NOT wrap the actual query in quotation marks -- Include all important information, examples, and constraints from the original query -- Keep questions comprehensive, prioritising easy understanding for end users - -## Example -Input: "I have a ferrari and my key battery has run out, is it possible that I can unlock it any other way? Can I use the key physically? Is there a number I can ring? My reg is 123456, is this important" - -Output: - -* Provide all possible ways of unlocking a Ferrari besides using a wireless key. -1. Inform physical key possibilities. -2. Provide Ferrari contact number. -3. Is proving the registration important (reg number - 123456) - +<|start_header_id|>assistant<|end_header_id|> diff --git a/packages/cdk/prompts/systemPrompt.txt b/packages/cdk/prompts/systemPrompt.txt index 4a7ec2f50..1eff0f5a4 100644 --- a/packages/cdk/prompts/systemPrompt.txt +++ b/packages/cdk/prompts/systemPrompt.txt @@ -5,14 +5,16 @@ STYLE & FORMATTING RULES: - Do NOT refer to the search results by number or name in the body of the text. - Do NOT add a "Citations" section at the end of the response. - Do NOT reference how the information was found (e.g., "...the provided search results") -- Text should prioritie readability. +- Do NOT state what the data is related to (e.g., "The search results are related to NHS API and FHIR...") +- Text should prioritise readability. - Links should use Markdown text, e.g., . - Use `Inline Code` for system names, field names, or technical terms (e.g., `HL7 FHIR`). STEPS: -1. Generate an answer, capturing the core question the user is asking. -2. Answer, directly, any individual or sub-questions the user has provided. -3. You must create a very short summary encapsulating the response and have it precede all other answers. +1. Extract key information from the knowledge base +2. Generate an answer, capturing the core question the user is asking. +3. Answer, directly, any individual or sub-questions the user has provided. +4. You must create a very short summary encapsulating the response and have it precede all other answers. EXAMPLE: diff --git a/packages/slackBotFunction/app/services/ai_processor.py b/packages/slackBotFunction/app/services/ai_processor.py index 1b5cb4355..1e0cfca1b 100644 --- a/packages/slackBotFunction/app/services/ai_processor.py +++ b/packages/slackBotFunction/app/services/ai_processor.py @@ -22,7 +22,7 @@ def process_ai_query(user_query: str, session_id: str | None = None) -> AIProces config.ORCHESTRATION_PROMPT_NAME, config.ORCHESTRATION_PROMPT_VERSION, ) - orchestrated_prompt = query_bedrock(user_query, orchestration_prompt_template, config, session_id) + orchestrated_prompt = query_bedrock(user_query, orchestration_prompt_template, config) orchestrated_text = orchestrated_prompt["output"]["text"] logger.debug("Orchestrated_text", extra={"text": orchestrated_text}) From cacb8d3cd3cf074282f065a42660223b0247491a Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Mon, 16 Feb 2026 16:21:02 +0000 Subject: [PATCH 16/21] feat: No sessionId for orchestration --- .../cdk/resources/BedrockPromptResources.ts | 14 +++++----- .../cdk/resources/BedrockPromptSettings.ts | 16 ++++++------ packages/cdk/resources/Functions.ts | 12 ++++----- packages/cdk/resources/RuntimePolicies.ts | 4 +-- packages/cdk/stacks/EpsAssistMeStack.ts | 10 +++---- packages/slackBotFunction/app/core/config.py | 12 ++++----- .../app/services/ai_processor.py | 17 ++++++------ packages/slackBotFunction/tests/conftest.py | 6 ++--- .../tests/test_ai_processor.py | 26 ++++++------------- 9 files changed, 54 insertions(+), 63 deletions(-) diff --git a/packages/cdk/resources/BedrockPromptResources.ts b/packages/cdk/resources/BedrockPromptResources.ts index 37ce1b09b..9274ebf39 100644 --- a/packages/cdk/resources/BedrockPromptResources.ts +++ b/packages/cdk/resources/BedrockPromptResources.ts @@ -15,7 +15,7 @@ export interface BedrockPromptResourcesProps { } export class BedrockPromptResources extends Construct { - public readonly orchestrationPrompt: Prompt + public readonly reformulationPrompt: Prompt public readonly ragResponsePrompt: Prompt public readonly modelId: string @@ -25,14 +25,14 @@ export class BedrockPromptResources extends Construct { const aiModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0") // Create Prompts - this.orchestrationPrompt = this.createPrompt( - "OrchestrationPrompt", - `${props.stackName}-Orchestration`, - "Prompt for orchestrating queries to improve RAG inference", + this.reformulationPrompt = this.createPrompt( + "ReformulationPrompt", + `${props.stackName}-reformulation`, + "Prompt for reformulation queries to improve RAG inference", aiModel, "", - [props.settings.orchestrationPrompt], - props.settings.orchestrationInferenceConfig + [props.settings.reformulationPrompt], + props.settings.reformulationInferenceConfig ) this.ragResponsePrompt = this.createPrompt( diff --git a/packages/cdk/resources/BedrockPromptSettings.ts b/packages/cdk/resources/BedrockPromptSettings.ts index c99bf7a76..70870b432 100644 --- a/packages/cdk/resources/BedrockPromptSettings.ts +++ b/packages/cdk/resources/BedrockPromptSettings.ts @@ -3,18 +3,18 @@ import {ChatMessage} from "@cdklabs/generative-ai-cdk-constructs/lib/cdk-lib/bed import {Construct} from "constructs" import {CfnPrompt} from "aws-cdk-lib/aws-bedrock" -export type BedrockPromptSettingsType = "system" | "orchestration" | "user" +export type BedrockPromptSettingsType = "system" | "reformulation" | "user" /** BedrockPromptSettings is responsible for loading and providing - * the system, user, and orchestration prompts along with their + * the system, user, and reformulation prompts along with their * inference configurations. */ export class BedrockPromptSettings extends Construct { public readonly systemPrompt: ChatMessage public readonly userPrompt: ChatMessage - public readonly orchestrationPrompt: ChatMessage + public readonly reformulationPrompt: ChatMessage public readonly ragInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty - public readonly orchestrationInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty + public readonly reformulationInferenceConfig: CfnPrompt.PromptModelInferenceConfigurationProperty /** * @param scope The Construct scope @@ -30,8 +30,8 @@ export class BedrockPromptSettings extends Construct { const userPromptData = this.getTypedPrompt("user") this.userPrompt = ChatMessage.user(userPromptData.text) - const orchestrationPrompt = this.getTypedPrompt("orchestration") - this.orchestrationPrompt = ChatMessage.assistant(orchestrationPrompt.text) + const reformulationPrompt = this.getTypedPrompt("reformulation") + this.reformulationPrompt = ChatMessage.assistant(reformulationPrompt.text) this.ragInferenceConfig = { temperature: 0, @@ -42,7 +42,7 @@ export class BedrockPromptSettings extends Construct { ] } - this.orchestrationInferenceConfig = { + this.reformulationInferenceConfig = { temperature: 0, topP: 0.3, maxTokens: 512, @@ -56,7 +56,7 @@ export class BedrockPromptSettings extends Construct { * If a version is provided, it retrieves that specific version. * Otherwise, it retrieves the latest version based on file naming. * - * @param type The type of prompt (system, user, orchestration) + * @param type The type of prompt (system, user, reformulation) * @returns An object containing the prompt text and filename */ private getTypedPrompt(type: BedrockPromptSettingsType) diff --git a/packages/cdk/resources/Functions.ts b/packages/cdk/resources/Functions.ts index a648211d9..d2463920f 100644 --- a/packages/cdk/resources/Functions.ts +++ b/packages/cdk/resources/Functions.ts @@ -28,14 +28,14 @@ export interface FunctionsProps { readonly slackBotTokenSecret: Secret readonly slackBotSigningSecret: Secret readonly slackBotStateTable: TableV2 - readonly orchestrationPromptName: string + readonly reformulationPromptName: string readonly ragResponsePromptName: string - readonly orchestrationPromptVersion: string + readonly reformulationPromptVersion: string readonly ragResponsePromptVersion: string readonly isPullRequest: boolean readonly mainSlackBotLambdaExecutionRoleArn : string readonly ragModelId: string - readonly orchestrationModelId: string + readonly reformulationModelId: string readonly notifyS3UploadFunctionPolicy: ManagedPolicy readonly docsBucketName: string } @@ -61,7 +61,7 @@ export class Functions extends Construct { dependencyLocation: ".dependencies/slackBotFunction", environmentVariables: { "RAG_MODEL_ID": props.ragModelId, - "ORCHESTRATION_MODEL_ID": props.orchestrationModelId, + "REFORMULATION_MODEL_ID": props.reformulationModelId, "KNOWLEDGEBASE_ID": props.knowledgeBaseId, "LAMBDA_MEMORY_SIZE": LAMBDA_MEMORY_SIZE, "SLACK_BOT_TOKEN_PARAMETER": props.slackBotTokenParameter.parameterName, @@ -69,9 +69,9 @@ export class Functions extends Construct { "GUARD_RAIL_ID": props.guardrailId, "GUARD_RAIL_VERSION": props.guardrailVersion, "SLACK_BOT_STATE_TABLE": props.slackBotStateTable.tableName, - "ORCHESTRATION_RESPONSE_PROMPT_NAME": props.orchestrationPromptName, + "REFORMULATION_RESPONSE_PROMPT_NAME": props.reformulationPromptName, "RAG_RESPONSE_PROMPT_NAME": props.ragResponsePromptName, - "ORCHESTRATION_RESPONSE_PROMPT_VERSION": props.orchestrationPromptVersion, + "REFORMULATION_RESPONSE_PROMPT_VERSION": props.reformulationPromptVersion, "RAG_RESPONSE_PROMPT_VERSION": props.ragResponsePromptVersion } }) diff --git a/packages/cdk/resources/RuntimePolicies.ts b/packages/cdk/resources/RuntimePolicies.ts index dda752df7..4250aac37 100644 --- a/packages/cdk/resources/RuntimePolicies.ts +++ b/packages/cdk/resources/RuntimePolicies.ts @@ -13,7 +13,7 @@ export interface RuntimePoliciesProps { readonly dataSourceArn: string readonly promptName: string readonly ragModelId: string - readonly orchestrationModelId: string + readonly reformulationModelId: string readonly docsBucketArn: string readonly docsBucketKmsKeyArn: string } @@ -32,7 +32,7 @@ export class RuntimePolicies extends Construct { actions: ["bedrock:InvokeModel"], resources: [ `arn:aws:bedrock:${props.region}::foundation-model/${props.ragModelId}`, - `arn:aws:bedrock:${props.region}::foundation-model/${props.orchestrationModelId}` + `arn:aws:bedrock:${props.region}::foundation-model/${props.reformulationModelId}` ] }) diff --git a/packages/cdk/stacks/EpsAssistMeStack.ts b/packages/cdk/stacks/EpsAssistMeStack.ts index 272a2cb97..468f6c8df 100644 --- a/packages/cdk/stacks/EpsAssistMeStack.ts +++ b/packages/cdk/stacks/EpsAssistMeStack.ts @@ -163,9 +163,9 @@ export class EpsAssistMeStack extends Stack { knowledgeBaseArn: vectorKB.knowledgeBase.attrKnowledgeBaseArn, guardrailArn: vectorKB.guardrail.guardrailArn, dataSourceArn: vectorKB.dataSourceArn, - promptName: bedrockPromptResources.orchestrationPrompt.promptName, + promptName: bedrockPromptResources.reformulationPrompt.promptName, ragModelId: bedrockPromptResources.modelId, - orchestrationModelId: bedrockPromptResources.modelId, + reformulationModelId: bedrockPromptResources.modelId, docsBucketArn: storage.kbDocsBucket.bucketArn, docsBucketKmsKeyArn: storage.kbDocsKmsKey.keyArn }) @@ -192,12 +192,12 @@ export class EpsAssistMeStack extends Stack { slackBotTokenSecret: secrets.slackBotTokenSecret, slackBotSigningSecret: secrets.slackBotSigningSecret, slackBotStateTable: tables.slackBotStateTable.table, - orchestrationPromptName: bedrockPromptResources.orchestrationPrompt.promptName, + reformulationPromptName: bedrockPromptResources.reformulationPrompt.promptName, ragResponsePromptName: bedrockPromptResources.ragResponsePrompt.promptName, - orchestrationPromptVersion: bedrockPromptResources.orchestrationPrompt.promptVersion, + reformulationPromptVersion: bedrockPromptResources.reformulationPrompt.promptVersion, ragResponsePromptVersion: bedrockPromptResources.ragResponsePrompt.promptVersion, ragModelId: bedrockPromptResources.modelId, - orchestrationModelId: bedrockPromptResources.modelId, + reformulationModelId: bedrockPromptResources.modelId, isPullRequest: isPullRequest, mainSlackBotLambdaExecutionRoleArn: mainSlackBotLambdaExecutionRoleArn, notifyS3UploadFunctionPolicy: runtimePolicies.notifyS3UploadFunctionPolicy, diff --git a/packages/slackBotFunction/app/core/config.py b/packages/slackBotFunction/app/core/config.py index 6cd591495..1127a225e 100644 --- a/packages/slackBotFunction/app/core/config.py +++ b/packages/slackBotFunction/app/core/config.py @@ -81,8 +81,8 @@ def get_retrieve_generate_config() -> BedrockConfig: GUARD_VERSION = os.environ["GUARD_RAIL_VERSION"] RAG_RESPONSE_PROMPT_NAME = os.environ["RAG_RESPONSE_PROMPT_NAME"] RAG_RESPONSE_PROMPT_VERSION = os.environ["RAG_RESPONSE_PROMPT_VERSION"] - ORCHESTRATION_RESPONSE_PROMPT_NAME = os.environ["ORCHESTRATION_RESPONSE_PROMPT_NAME"] - ORCHESTRATION_RESPONSE_PROMPT_VERSION = os.environ["ORCHESTRATION_RESPONSE_PROMPT_VERSION"] + REFORMULATION_RESPONSE_PROMPT_NAME = os.environ["REFORMULATION_RESPONSE_PROMPT_NAME"] + REFORMULATION_RESPONSE_PROMPT_VERSION = os.environ["REFORMULATION_RESPONSE_PROMPT_VERSION"] logger.info( "Guardrail configuration loaded", extra={"guardrail_id": GUARD_RAIL_ID, "guardrail_version": GUARD_VERSION} @@ -96,8 +96,8 @@ def get_retrieve_generate_config() -> BedrockConfig: GUARD_VERSION, RAG_RESPONSE_PROMPT_NAME, RAG_RESPONSE_PROMPT_VERSION, - ORCHESTRATION_RESPONSE_PROMPT_NAME, - ORCHESTRATION_RESPONSE_PROMPT_VERSION, + REFORMULATION_RESPONSE_PROMPT_NAME, + REFORMULATION_RESPONSE_PROMPT_VERSION, ) @@ -152,8 +152,8 @@ class BedrockConfig: GUARD_VERSION: str RAG_RESPONSE_PROMPT_NAME: str RAG_RESPONSE_PROMPT_VERSION: str - ORCHESTRATION_PROMPT_NAME: str - ORCHESTRATION_PROMPT_VERSION: str + REFORMULATION_PROMPT_NAME: str + REFORMULATION_PROMPT_VERSION: str @dataclass diff --git a/packages/slackBotFunction/app/services/ai_processor.py b/packages/slackBotFunction/app/services/ai_processor.py index 1e0cfca1b..d7f4a07f9 100644 --- a/packages/slackBotFunction/app/services/ai_processor.py +++ b/packages/slackBotFunction/app/services/ai_processor.py @@ -2,7 +2,7 @@ shared AI processing service - extracted to avoid duplication both slack handlers and direct invocation use identical logic for query -orchestration and bedrock interaction. single source of truth for AI flows. +reformulation and bedrock interaction. single source of truth for AI flows. """ from app.services.bedrock import query_bedrock @@ -18,17 +18,18 @@ def process_ai_query(user_query: str, session_id: str | None = None) -> AIProces # session_id enables conversation continuity across multiple queries config = get_retrieve_generate_config() - orchestration_prompt_template = load_prompt( - config.ORCHESTRATION_PROMPT_NAME, - config.ORCHESTRATION_PROMPT_VERSION, + reformulation_prompt_template = load_prompt( + config.REFORMULATION_PROMPT_NAME, + config.REFORMULATION_PROMPT_VERSION, ) - orchestrated_prompt = query_bedrock(user_query, orchestration_prompt_template, config) - orchestrated_text = orchestrated_prompt["output"]["text"] + # Don't provide sessionId as this conflicts with the sessions knowledgebase settings + reformulation_prompt = query_bedrock(user_query, reformulation_prompt_template, config) + reformulation_text = reformulation_prompt["output"]["text"] - logger.debug("Orchestrated_text", extra={"text": orchestrated_text}) + logger.debug("reformulation_text", extra={"text": reformulation_text}) rag_prompt_template = load_prompt(config.RAG_RESPONSE_PROMPT_NAME, config.RAG_RESPONSE_PROMPT_VERSION) - kb_response = query_bedrock(orchestrated_text, rag_prompt_template, config, session_id) + kb_response = query_bedrock(reformulation_text, rag_prompt_template, config, session_id) logger.info( "response from bedrock", diff --git a/packages/slackBotFunction/tests/conftest.py b/packages/slackBotFunction/tests/conftest.py index 4264f051c..e8795660b 100644 --- a/packages/slackBotFunction/tests/conftest.py +++ b/packages/slackBotFunction/tests/conftest.py @@ -20,9 +20,9 @@ def mock_env(): "AWS_REGION": "eu-west-2", "GUARD_RAIL_ID": "test-guard-id", "GUARD_RAIL_VERSION": "1", - "ORCHESTRATION_MODEL_ID": "test-model", - "ORCHESTRATION_RESPONSE_PROMPT_NAME": "test-prompt", - "ORCHESTRATION_RESPONSE_PROMPT_VERSION": "DRAFT", + "REFORMULATION_MODEL_ID": "test-model", + "REFORMULATION_RESPONSE_PROMPT_NAME": "test-prompt", + "REFORMULATION_RESPONSE_PROMPT_VERSION": "DRAFT", "RAG_MODEL_ID": "test-model-id", "RAG_RESPONSE_PROMPT_NAME": "test-rag-prompt", "RAG_RESPONSE_PROMPT_VERSION": "DRAFT", diff --git a/packages/slackBotFunction/tests/test_ai_processor.py b/packages/slackBotFunction/tests/test_ai_processor.py index 7ded9be88..e971f4687 100644 --- a/packages/slackBotFunction/tests/test_ai_processor.py +++ b/packages/slackBotFunction/tests/test_ai_processor.py @@ -1,4 +1,4 @@ -"""shared ai processor - validates query orchestration and bedrock integration""" +"""shared ai processor - validates query reformulation and bedrock integration""" import pytest from unittest.mock import call, patch, ANY @@ -10,8 +10,8 @@ def mock_config_setup(mock_load_prompt, mock_config): """Setup common mock configurations""" mock_load_prompt.return_value = {"prompt_text": "test_prompt", "model_id": "model_id", "inference_config": {}} mock_config.get_retrieve_generate_config.return_value = { - "ORCHESTRATION_PROMPT_NAME": "test", - "ORCHESTRATION_PROMPT_VERSION": "test", + "REFORMULATION_PROMPT_NAME": "test", + "REFORMULATION_PROMPT_VERSION": "test", "RAG_RESPONSE_PROMPT_NAME": "test", "RAG_RESPONSE_PROMPT_VERSION": "test", } @@ -44,13 +44,8 @@ def test_process_ai_query_without_session(self, mock_bedrock, mock_load_prompt, mock_bedrock.assert_has_calls( [ - call("How to authenticate EPS API?", mock_load_prompt.return_value, ANY, None), - call( - "To authenticate with EPS API, you need...", - mock_load_prompt.return_value, - ANY, - None, - ), + call("How to authenticate EPS API?", mock_load_prompt.return_value, ANY), + call("To authenticate with EPS API, you need...", mock_load_prompt.return_value, ANY, None), ] ) @@ -76,13 +71,8 @@ def test_process_ai_query_with_session(self, mock_bedrock, mock_load_prompt, moc mock_bedrock.assert_has_calls( [ - call("What about rate limits?", mock_load_prompt.return_value, ANY, mock_session_id), - call( - "EPS API has rate limits of...", - mock_load_prompt.return_value, - ANY, - mock_session_id, - ), + call("What about rate limits?", mock_load_prompt.return_value, ANY), + call("EPS API has rate limits of...", mock_load_prompt.return_value, ANY, mock_session_id), ] ) @@ -149,7 +139,7 @@ def test_process_ai_query_empty_query(self, mock_bedrock, mock_load_prompt, mock mock_bedrock.assert_has_calls( [ - call("", ANY, ANY, None), + call("", ANY, ANY), call("Please provide a question", ANY, ANY, None), ] ) From 67e3220d2ffaaffe3a626e34883c8fc51767c220 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Mon, 16 Feb 2026 16:25:13 +0000 Subject: [PATCH 17/21] feat: Rename orchestration back to reformulation --- .../prompts/{orchestrationPrompt.txt => reformulationPrompt.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/cdk/prompts/{orchestrationPrompt.txt => reformulationPrompt.txt} (100%) diff --git a/packages/cdk/prompts/orchestrationPrompt.txt b/packages/cdk/prompts/reformulationPrompt.txt similarity index 100% rename from packages/cdk/prompts/orchestrationPrompt.txt rename to packages/cdk/prompts/reformulationPrompt.txt From 3d60d29ffdd147529a365c96696ddc980df70846 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Mon, 16 Feb 2026 17:35:21 +0000 Subject: [PATCH 18/21] feat: Reduce max tokens, to reduce hallucinations --- packages/cdk/prompts/reformulationPrompt.txt | 6 ------ packages/cdk/prompts/systemPrompt.txt | 6 +++++- packages/cdk/prompts/userPrompt.txt | 3 --- packages/cdk/resources/BedrockPromptSettings.ts | 14 ++++---------- 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/packages/cdk/prompts/reformulationPrompt.txt b/packages/cdk/prompts/reformulationPrompt.txt index 615df133f..eb6e5039a 100644 --- a/packages/cdk/prompts/reformulationPrompt.txt +++ b/packages/cdk/prompts/reformulationPrompt.txt @@ -13,9 +13,6 @@ Output your response strictly using the following XML structure: <|eot_id|> <|start_header_id|>user<|end_header_id|> -### Search Context -The company leave policy dictates that standard PTO accrues at 1.5 days per month. However, employees on probation (first 90 days) cannot use accrued PTO. If an employee transitions from part-time to full-time, their prior accrued PTO carries over, but the new accrual rate of 1.5 days begins on the official transition date. Negative PTO balances are only permitted for full-time employees with at least 1 year of tenure, up to a maximum of -3 days. - ### User Query Hi, I need some help figuring out the PTO rules for one of my team members. They started as part-time 6 months ago, but they just transitioned to full-time last week (let's say exactly 7 days ago). They currently have 2 days of PTO saved up from their part-time stint. They want to take next week off entirely, which would require 5 days of PTO. Can they do this, effectively going to a -3 balance, since they are full-time now? <|eot_id|> @@ -30,9 +27,6 @@ Can a recently transitioned full-time employee with 6 months total tenure and 2 - Current PTO Balance: 2 days - Requested PTO: 5 days (resulting in -3 balance) - -Part-time PTO carries over upon transitioning to full-time. Negative PTO balances (up to -3 days) are strictly reserved for full-time employees with at least 1 year of tenure. - <|eot_id|> <|start_header_id|>user<|end_header_id|> diff --git a/packages/cdk/prompts/systemPrompt.txt b/packages/cdk/prompts/systemPrompt.txt index 1eff0f5a4..2b0189b4b 100644 --- a/packages/cdk/prompts/systemPrompt.txt +++ b/packages/cdk/prompts/systemPrompt.txt @@ -1,5 +1,5 @@ You are a technical assistant specialized in onboarding guidance. -Your primary goal is to answer questions using ONLY the provided search results. +Your primary goal is to STYLE & FORMATTING RULES: - Do NOT refer to the search results by number or name in the body of the text. @@ -10,6 +10,10 @@ STYLE & FORMATTING RULES: - Links should use Markdown text, e.g., . - Use `Inline Code` for system names, field names, or technical terms (e.g., `HL7 FHIR`). +RULES: +- Answer questions using ONLY the provided search results. +- Do not assume any information, all information must be grounded in data. + STEPS: 1. Extract key information from the knowledge base 2. Generate an answer, capturing the core question the user is asking. diff --git a/packages/cdk/prompts/userPrompt.txt b/packages/cdk/prompts/userPrompt.txt index 50ed2db93..76c6d6140 100644 --- a/packages/cdk/prompts/userPrompt.txt +++ b/packages/cdk/prompts/userPrompt.txt @@ -1,6 +1,3 @@ - $search_results$ {{user_query}} - -Answers: diff --git a/packages/cdk/resources/BedrockPromptSettings.ts b/packages/cdk/resources/BedrockPromptSettings.ts index 70870b432..096650b68 100644 --- a/packages/cdk/resources/BedrockPromptSettings.ts +++ b/packages/cdk/resources/BedrockPromptSettings.ts @@ -33,16 +33,7 @@ export class BedrockPromptSettings extends Construct { const reformulationPrompt = this.getTypedPrompt("reformulation") this.reformulationPrompt = ChatMessage.assistant(reformulationPrompt.text) - this.ragInferenceConfig = { - temperature: 0, - topP: 0.1, - maxTokens: 1024, - stopSequences: [ - "Human:" - ] - } - - this.reformulationInferenceConfig = { + const defaultInferenceConfig = { temperature: 0, topP: 0.3, maxTokens: 512, @@ -50,6 +41,9 @@ export class BedrockPromptSettings extends Construct { "Human:" ] } + + this.ragInferenceConfig = defaultInferenceConfig + this.reformulationInferenceConfig = defaultInferenceConfig } /** Get the latest prompt text from files in the specified directory. From 2154fcc4c48bcaa9e2cc7d8dd853739efc5842e1 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Tue, 17 Feb 2026 08:43:33 +0000 Subject: [PATCH 19/21] feat: Remove automatic bullet point formatting --- .../app/slack/slack_events.py | 10 +- .../test_slack_events_citations.py | 73 ---- .../test_slack_events_messages.py | 363 +++++++++++++++++- 3 files changed, 367 insertions(+), 79 deletions(-) diff --git a/packages/slackBotFunction/app/slack/slack_events.py b/packages/slackBotFunction/app/slack/slack_events.py index 726968cc6..46adcdbb7 100644 --- a/packages/slackBotFunction/app/slack/slack_events.py +++ b/packages/slackBotFunction/app/slack/slack_events.py @@ -281,10 +281,14 @@ def convert_markdown_to_slack(body: str) -> str: body = re.sub(r"([\*_]){2,10}([^*]+)([\*_]){2,10}", r"\1\2\1", body) # 3. Handle Lists (Handle various bullet points and dashes, inc. unicode support) - body = re.sub(r"(?:^|\s{1,10})[-•–—▪‣◦⁃]\s{0,10}", r"\n- ", body) + body = re.sub(r"[-•–—▪‣◦⁃]", r"-", body) - # 4. Convert Markdown Links [text](url) to Slack - body = re.sub(r"\[([^\]]+)\]\(([^\)]+)\)", r"<\2|\1>", body) + # 3. Convert Markdown Links [text](url) to Slack + matches = re.findall(r"\[([^\]]+)\]\(([^\)]+)\)", body) + + for match in matches: + text, url = match + body = body.replace(f"[{text}]({url})", f"<{url}|{text.replace('\n', ' ').replace(r"/n{2,}", ' ')[:50]}>") return body.strip() diff --git a/packages/slackBotFunction/tests/test_slack_events/test_slack_events_citations.py b/packages/slackBotFunction/tests/test_slack_events/test_slack_events_citations.py index f054b483a..b494c0875 100644 --- a/packages/slackBotFunction/tests/test_slack_events/test_slack_events_citations.py +++ b/packages/slackBotFunction/tests/test_slack_events/test_slack_events_citations.py @@ -560,79 +560,6 @@ def test_create_response_body_creates_body_with_markdown_formatting( assert "*Bold*, _italics_, and `code`." in citation_value.get("body") -def test_create_response_body_creates_body_with_lists( - mock_get_parameter: Mock, - mock_env: Mock, -): - """Test regex text processing functionality within process_async_slack_event""" - # delete and import module to test - if "app.slack.slack_events" in sys.modules: - del sys.modules["app.slack.slack_events"] - from app.slack.slack_events import _create_response_body - - dirty_input = "Header text - Standard Dash -No Space Dash • Standard Bullet -NoSpace-NoSpace" - - # perform operation - response = _create_response_body( - citations=[ - { - "source_number": "1", - "title": "Citation Title", - "excerpt": dirty_input, - "relevance_score": "0.95", - } - ], - feedback_data={}, - response_text="This is a response with a citation.[1]", - ) - - # assertions - assert len(response) > 1 - assert response[1]["type"] == "actions" - assert response[1]["block_id"] == "citation_actions" - - citation_element = response[1]["elements"][0] - citation_value = json.loads(citation_element["value"]) - - expected_output = "Header text\n- Standard Dash\n- No Space Dash\n- Standard Bullet\n- NoSpace-NoSpace" - assert expected_output in citation_value.get("body") - - -def test_create_response_body_creates_body_without_encoding_errors( - mock_get_parameter: Mock, - mock_env: Mock, -): - """Test regex text processing functionality within process_async_slack_event""" - # delete and import module to test - if "app.slack.slack_events" in sys.modules: - del sys.modules["app.slack.slack_events"] - from app.slack.slack_events import _create_response_body - - # perform operation - response = _create_response_body( - citations=[ - { - "source_number": "1", - "title": "Citation Title", - "excerpt": "» Tabbing Issue. ⢠Bullet point issue.", - "relevance_score": "0.95", - } - ], - feedback_data={}, - response_text="This is a response with a citation.[1]", - ) - - # assertions - assert len(response) > 1 - assert response[1]["type"] == "actions" - assert response[1]["block_id"] == "citation_actions" - - citation_element = response[1]["elements"][0] - citation_value = json.loads(citation_element["value"]) - - assert "Tabbing Issue.\n- Bullet point issue." in citation_value.get("body") - - @patch("app.services.ai_processor.process_ai_query") def test_create_citation_logs_citations( mock_process_ai_query: Mock, diff --git a/packages/slackBotFunction/tests/test_slack_events/test_slack_events_messages.py b/packages/slackBotFunction/tests/test_slack_events/test_slack_events_messages.py index 60d52c7df..0a142ce54 100644 --- a/packages/slackBotFunction/tests/test_slack_events/test_slack_events_messages.py +++ b/packages/slackBotFunction/tests/test_slack_events/test_slack_events_messages.py @@ -466,7 +466,7 @@ def test_create_response_body_creates_body_with_lists( del sys.modules["app.slack.slack_events"] from app.slack.slack_events import _create_response_body - dirty_input = "Header text - Standard Dash -No Space Dash • Standard Bullet -NoSpace-NoSpace" + dirty_input = "Header text - Standard Dash -No Space Dash • Standard Bullet -DoubleSpace-NoSpace" # perform operation response = _create_response_body( @@ -481,7 +481,7 @@ def test_create_response_body_creates_body_with_lists( response_value = response[0]["text"]["text"] - expected_output = "Header text\n- Standard Dash\n- No Space Dash\n- Standard Bullet\n- NoSpace-NoSpace" + expected_output = "Header text - Standard Dash -No Space Dash - Standard Bullet -DoubleSpace-NoSpace" assert expected_output in response_value @@ -508,4 +508,361 @@ def test_create_response_body_creates_body_without_encoding_errors( response_value = response[0]["text"]["text"] - assert "Tabbing Issue.\n- Bullet point issue." in response_value + assert "Tabbing Issue. - Bullet point issue." in response_value + + +# ================================================================ +# Tests for _create_citation +# ================================================================ + + +def test_create_citation_high_relevance_score(mock_get_parameter: Mock, mock_env: Mock): + """Test citation creation with high relevance score""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + citation = { + "source_number": "1", + "title": "Test Document", + "excerpt": "This is a test excerpt", + "relevance_score": "0.95", + } + feedback_data = {"ck": "conv-123", "ch": "C789"} + response_text = "Response with [cit_1] citation" + + result = _create_citation(citation, feedback_data, response_text) + + assert len(result["action_buttons"]) == 1 + button = result["action_buttons"][0] + assert button["type"] == "button" + assert button["text"]["text"] == "[1] Test Document" + assert button["action_id"] == "cite_1" + assert result["response_text"] == "Response with [1] citation" + + +def test_create_citation_low_relevance_score(mock_get_parameter: Mock, mock_env: Mock): + """Test citation is skipped when relevance score is low""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + citation = { + "source_number": "2", + "title": "Low Relevance Doc", + "excerpt": "This is low relevance", + "relevance_score": "0.5", # Below 0.6 threshold + } + feedback_data = {"ck": "conv-123"} + response_text = "Response with [cit_2]" + + result = _create_citation(citation, feedback_data, response_text) + + assert len(result["action_buttons"]) == 0 + assert result["response_text"] == "Response with [cit_2]" + + +def test_create_citation_missing_excerpt(mock_get_parameter: Mock, mock_env: Mock): + """Test citation with missing excerpt uses default message""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + citation = { + "source_number": "3", + "title": "Document Without Excerpt", + "relevance_score": "0.9", + } + feedback_data = {"ck": "conv-123"} + response_text = "Response text" + + result = _create_citation(citation, feedback_data, response_text) + + assert len(result["action_buttons"]) == 1 + import json + + button_data = json.loads(result["action_buttons"][0]["value"]) + assert button_data["body"] == "No document excerpt available." + + +def test_create_citation_missing_title_uses_filename(mock_get_parameter: Mock, mock_env: Mock): + """Test citation uses filename when title is missing""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + citation = { + "source_number": "4", + "filename": "document.pdf", + "excerpt": "Some content", + "relevance_score": "0.85", + } + feedback_data = {"ck": "conv-123"} + response_text = "Response text" + + result = _create_citation(citation, feedback_data, response_text) + + assert len(result["action_buttons"]) == 1 + assert result["action_buttons"][0]["text"]["text"] == "[4] document.pdf" + + +def test_create_citation_fallback_source_when_missing(mock_get_parameter: Mock, mock_env: Mock): + """Test citation uses 'Source' when both title and filename are missing""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + citation = { + "source_number": "5", + "excerpt": "Some content", + "relevance_score": "0.9", + } + feedback_data = {"ck": "conv-123"} + response_text = "Response text" + + result = _create_citation(citation, feedback_data, response_text) + + assert len(result["action_buttons"]) == 1 + assert result["action_buttons"][0]["text"]["text"] == "[5] Source" + + +def test_create_citation_button_text_truncation(mock_get_parameter: Mock, mock_env: Mock): + """Test citation button text is truncated when exceeds 75 characters""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + long_title = "A" * 80 # Title longer than 75 chars + citation = { + "source_number": "6", + "title": long_title, + "excerpt": "Some content", + "relevance_score": "0.9", + } + feedback_data = {"ck": "conv-123"} + response_text = "Response text" + + result = _create_citation(citation, feedback_data, response_text) + + button_text = result["action_buttons"][0]["text"]["text"] + assert len(button_text) <= 77 # "[X] " + 70 chars + "..." + assert button_text.endswith("...") + + +def test_create_citation_removes_newlines_from_source_number(mock_get_parameter: Mock, mock_env: Mock): + """Test citation removes newlines from source number""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + citation = { + "source_number": "7\n\n", + "title": "Test", + "excerpt": "Content", + "relevance_score": "0.9", + } + feedback_data = {"ck": "conv-123"} + response_text = "Response with [cit_7]" + + result = _create_citation(citation, feedback_data, response_text) + + assert result["response_text"] == "Response with [7]" + assert result["action_buttons"][0]["action_id"] == "cite_7" + + +def test_create_citation_zero_relevance_score(mock_get_parameter: Mock, mock_env: Mock): + """Test citation with zero relevance score is skipped""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import _create_citation + + citation = { + "source_number": "8", + "title": "No Relevance", + "excerpt": "Content", + "relevance_score": "0", + } + feedback_data = {"ck": "conv-123"} + response_text = "Response text" + + result = _create_citation(citation, feedback_data, response_text) + + assert len(result["action_buttons"]) == 0 + + +# ================================================================ +# Tests for convert_markdown_to_slack +# ================================================================ + + +def test_convert_markdown_to_slack_bold_formatting(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion of markdown bold to Slack formatting""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + markdown_text = "This is **bold text** in markdown" + result = convert_markdown_to_slack(markdown_text) + + assert "*bold text*" in result + + +def test_convert_markdown_to_slack_italic_formatting(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion of markdown italics to Slack formatting""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + markdown_text = "This is __italic text__ in markdown" + result = convert_markdown_to_slack(markdown_text) + + assert "_italic text_" in result + + +def test_convert_markdown_to_slack_links(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion of markdown links to Slack formatting""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + markdown_text = "Check out [this link](https://example.com)" + result = convert_markdown_to_slack(markdown_text) + + assert "" in result + + +def test_convert_markdown_to_slack_encoding_issues_arrow(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion removes arrow encoding issues""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text_with_encoding = "» Tab issue" + result = convert_markdown_to_slack(text_with_encoding) + + assert "»" not in result + assert "Tab issue" in result + + +def test_convert_markdown_to_slack_encoding_issues_bullet(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion fixes bullet point encoding issues""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text_with_encoding = "⢠Bullet point" + result = convert_markdown_to_slack(text_with_encoding) + + assert "â¢" not in result + assert "- Bullet point" in result + + +def test_convert_markdown_to_slack_empty_string(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion of empty string""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + result = convert_markdown_to_slack("") + + assert result == "" + + +def test_convert_markdown_to_slack_none_string(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion of None string""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + result = convert_markdown_to_slack(None) + + assert result == "" + + +def test_convert_markdown_to_slack_combined_formatting(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion with multiple formatting types""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text = "Check **bold** and __italic__ with [link](https://test.com)" + result = convert_markdown_to_slack(text) + + assert "*bold*" in result + assert "_italic_" in result + assert "" in result + + +def test_convert_markdown_to_slack_link_with_newlines_no_space(mock_get_parameter: Mock, mock_env: Mock): + """Test link conversion removes newlines from link text""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text = "Check [link\ntext](https://example.com)" + result = convert_markdown_to_slack(text) + + assert "" in result + + +def test_convert_markdown_to_slack_link_with_newlines_with_space(mock_get_parameter: Mock, mock_env: Mock): + """Test link conversion removes newlines from link text""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text = "Check [link \n text](https://example.com)" + result = convert_markdown_to_slack(text) + + assert "" in result + + +def test_convert_markdown_to_slack_link_with_newlines_with_dash(mock_get_parameter: Mock, mock_env: Mock): + """Test link conversion removes newlines from link text""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text = "Check [link-text](https://example.com)" + result = convert_markdown_to_slack(text) + + assert "" in result + + +def test_convert_markdown_to_slack_link_with_newlines_with_dash_and_space(mock_get_parameter: Mock, mock_env: Mock): + """Test link conversion removes newlines from link text""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text = "Check [link - text](https://example.com)" + result = convert_markdown_to_slack(text) + + assert "" in result + + +def test_convert_markdown_to_slack_whitespace_stripped(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion strips leading/trailing whitespace""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text = " Some text with spaces " + result = convert_markdown_to_slack(text) + + assert result == "Some text with spaces" + + +def test_convert_markdown_to_slack_multiple_encoding_issues(mock_get_parameter: Mock, mock_env: Mock): + """Test conversion handles multiple encoding issues""" + if "app.slack.slack_events" in sys.modules: + del sys.modules["app.slack.slack_events"] + from app.slack.slack_events import convert_markdown_to_slack + + text = "» Tab issue. ⢠Bullet point ⢠another bullet" + result = convert_markdown_to_slack(text) + + assert "»" not in result + assert "â¢" not in result + assert "- Bullet point" in result + assert "- another bullet" in result From 8b9b1d712021dc9fbc3ebd137198d93bf256d9df Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Tue, 17 Feb 2026 09:19:01 +0000 Subject: [PATCH 20/21] fix: floating point equality check --- packages/slackBotFunction/tests/test_bedrock_integration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/slackBotFunction/tests/test_bedrock_integration.py b/packages/slackBotFunction/tests/test_bedrock_integration.py index 75a72435b..24d87cd5a 100644 --- a/packages/slackBotFunction/tests/test_bedrock_integration.py +++ b/packages/slackBotFunction/tests/test_bedrock_integration.py @@ -1,3 +1,4 @@ +import math import sys from unittest.mock import Mock, patch, MagicMock, ANY @@ -119,4 +120,4 @@ def test_query_bedrock_check_config(mock_boto_client: Mock, mock_env: Mock): assert prompt_config["temperature"] == 0 assert prompt_config["maxTokens"] == 1024 - assert prompt_config["topP"] == 0.1 + assert math.isclose(prompt_config["topP"], 0.1, rel_tol=1e-09, abs_tol=1e-09) From 0b5aad7ebbc11e3c99a90f01dcf199edcaaf53e3 Mon Sep 17 00:00:00 2001 From: Kieran Wilkinson Date: Thu, 19 Feb 2026 09:14:49 +0000 Subject: [PATCH 21/21] feat: Use Fixed Size Chunking --- .../cdk/resources/VectorKnowledgeBaseResources.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/cdk/resources/VectorKnowledgeBaseResources.ts b/packages/cdk/resources/VectorKnowledgeBaseResources.ts index 7bb0bd160..8abd8e1b2 100644 --- a/packages/cdk/resources/VectorKnowledgeBaseResources.ts +++ b/packages/cdk/resources/VectorKnowledgeBaseResources.ts @@ -158,11 +158,10 @@ export class VectorKnowledgeBaseResources extends Construct { const chunkingConfiguration: CfnDataSource.ChunkingConfigurationProperty = { ...ChunkingStrategy.SEMANTIC.configuration, - semanticChunkingConfiguration: { - breakpointPercentileThreshold: 80, - bufferSize: 1, - maxTokens: 350 - } satisfies CfnDataSource.SemanticChunkingConfigurationProperty + fixedSizeChunkingConfiguration: { + maxTokens: 512, + overlapPercentage: 25 + } satisfies CfnDataSource.FixedSizeChunkingConfigurationProperty } const hash = crypto.createHash("md5") @@ -181,9 +180,6 @@ export class VectorKnowledgeBaseResources extends Construct { bucketArn: props.docsBucket.bucketArn, inclusionPrefixes: ["processed/"] } - }, - vectorIngestionConfiguration: { - chunkingConfiguration: chunkingConfiguration } })