diff --git a/scripts/audit/run_audit.sh b/scripts/audit/run_audit.sh index ea6e90f..23aaae2 100755 --- a/scripts/audit/run_audit.sh +++ b/scripts/audit/run_audit.sh @@ -4,16 +4,37 @@ set -e +USE_ADVISORY=false + +# Parse flags +while [[ "$1" == --* ]]; do + case "$1" in + --advisory) + USE_ADVISORY=true + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + if [ -z "$1" ]; then - echo "Usage: $0 "; + echo "Usage: $0 [--advisory] "; exit 1; fi python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.fetch_source_code -g repo="$1" python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.identify_applications -g repo="$1" python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.gather_web_entry_point_info -g repo="$1" -python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.classify_application_local -g repo="$1" -python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.audit_issue_local_iter -g repo="$1" + +if [ "$USE_ADVISORY" = true ]; then + python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.fetch_security_advisories -g repo="$1" +fi + +python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.classify_application_local -g repo="$1" -g use_advisory="$USE_ADVISORY" +python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.audit_issue_local_iter -g repo="$1" -g use_advisory="$USE_ADVISORY" set +e diff --git a/scripts/audit/run_audit_in_docker.sh b/scripts/audit/run_audit_in_docker.sh index 03d4a2c..9e99ea0 100755 --- a/scripts/audit/run_audit_in_docker.sh +++ b/scripts/audit/run_audit_in_docker.sh @@ -5,4 +5,4 @@ # https://stackoverflow.com/a/53122736 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -${__dir}/../run_in_docker.sh ${__dir}/run_audit.sh "$1" +${__dir}/../run_in_docker.sh ${__dir}/run_audit.sh "$@" diff --git a/src/seclab_taskflows/personalities/web_application_security_expert.yaml b/src/seclab_taskflows/personalities/web_application_security_expert.yaml index b2ae6b2..58033e0 100644 --- a/src/seclab_taskflows/personalities/web_application_security_expert.yaml +++ b/src/seclab_taskflows/personalities/web_application_security_expert.yaml @@ -20,3 +20,4 @@ toolboxes: - seclab_taskflow_agent.toolboxes.memcache - seclab_taskflows.toolboxes.gh_file_viewer - seclab_taskflow_agent.toolboxes.codeql + - seclab_taskflows.toolboxes.ghsa diff --git a/src/seclab_taskflows/prompts/audit/known_security_advisories.yaml b/src/seclab_taskflows/prompts/audit/known_security_advisories.yaml new file mode 100644 index 0000000..6c5cb11 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/known_security_advisories.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + ## Known Security Advisories for this Repository + + Fetch the security advisories for {{ globals.repo }} from memcache (stored under the key 'security_advisories_{{ globals.repo }}'). If the value in the memcache is null or an error message, clearly state that no advisories are available and skip advisory analysis. Otherwise, state how many advisories were found. + Review these advisories and consider them when identifying security risks. If you identify code that is similar to a known advisory pattern, highlight that connection. diff --git a/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml b/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml index 22fcc21..64a375f 100644 --- a/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml +++ b/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml @@ -8,6 +8,7 @@ model_config: seclab_taskflows.configs.model_config globals: repo: + use_advisory: # Taskflow to audit some potential issues. taskflow: - task: @@ -29,20 +30,24 @@ taskflow: - seclab_taskflows.personalities.web_application_security_expert model: code_analysis user_prompt: | - The issue is in repo {{ result.repo }} with id {{ result.issue_id }}. The component is under the directory + The issue is in repo {{ result.repo }} with id {{ result.issue_id }}. The component is under the directory {{ result.location }} with component_id {{ result.component_id }}. The notes of the component is: {{ result.component_notes }} - You should use this to understand the intended purpose of the component and take it into account when + You should use this to understand the intended purpose of the component and take it into account when you audit the issue. The type of the issue is {{ result.issue_type }} and here is the notes of the issue: {{ result.issue_notes }} + {% if globals.use_advisory == 'true' %} + {% include 'seclab_taskflows.prompts.audit.known_security_advisories' %} + {% endif %} + {% include 'seclab_taskflows.prompts.audit.audit_issue' %} toolboxes: - seclab_taskflows.toolboxes.repo_context - seclab_taskflows.toolboxes.local_file_viewer - \ No newline at end of file + - seclab_taskflow_agent.toolboxes.memcache diff --git a/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml b/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml index d1da08a..9f44ccc 100644 --- a/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml +++ b/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml @@ -8,6 +8,7 @@ model_config: seclab_taskflows.configs.model_config globals: repo: + use_advisory: # Taskflow to analyze the general contextual information of a project and classify the different applications within it taskflow: - task: @@ -36,8 +37,12 @@ taskflow: Fetch the entry points and web entry points of the component, then the user actions of this component. Based on the entry points, web entry points, components, user actions and README.md and if available, SECURITY.md in the {{ globals.repo }}, can you tell me what type of application this repo is and what kind of security boundary it has. - Based on this, determine whether the component is likely to have security problems. - + Based on this, determine whether the component is likely to have security problems. + + {% if globals.use_advisory == 'true' %} + {% include 'seclab_taskflows.prompts.audit.known_security_advisories' %} + {% endif %} + Identify the most likely security problems in the component. Your task is not to carry out a full audit, but to identify the main risk in the component so that further analysis can be carried out. Do not be too specific about an issue, but rather craft your report based on the general functionality and type of @@ -50,7 +55,7 @@ taskflow: - Is this component likely to take untrusted user input? For example, remote web requests or IPC, RPC calls? - What is the intended purpose of this component and its functionality? Does it allow high privileged actions? Is it intended to provide such functionalities for all users? Or is there complex access control logic involved? - - The component itself may also have its own `README.md` (or a subdirectory of it may have a `README.md`). Take + - The component itself may also have its own `README.md` (or a subdirectory of it may have a `README.md`). Take a look at those files to help understand the functionality of the component. For example, an Admin UI/dashboard may be susceptible to client side Javascript vulnerabilities such as XSS, CSRF. @@ -60,7 +65,7 @@ taskflow: a web frontend may allow users to access their own content and admins to access all content, but users should not be able to access another users' content in general. - We're looking for more concrete and serious security issues that affects system integrity or + We're looking for more concrete and serious security issues that affects system integrity or lead to information leak, so please do not include issues like brute force, Dos, log injection etc. Also do not include issues that require the system to be already compromised, such as issues that rely on malicious @@ -72,9 +77,9 @@ taskflow: Your task is to identify risk rather than properly audit and find security issues. Do not look too much into the implementation or scrutinize the security measures such as access control and sanitizers at this stage. Instead, report more general risks that are associated with the type of component - that you are looking at. + that you are looking at. - It is not your task to audit the security measures, but rather just to identify the risks and suggest some issues + It is not your task to audit the security measures, but rather just to identify the risks and suggest some issues that is worth auditing. Reflect on your notes and check that the attack scenario meets the above requirements. Exclude low severity issues or @@ -84,4 +89,5 @@ taskflow: If you think the issues satisfy the criteria, store a component issue entry for each type of issue identified. toolboxes: - seclab_taskflows.toolboxes.repo_context - - seclab_taskflows.toolboxes.local_file_viewer \ No newline at end of file + - seclab_taskflows.toolboxes.local_file_viewer + - seclab_taskflow_agent.toolboxes.memcache diff --git a/src/seclab_taskflows/taskflows/audit/fetch_security_advisories.yaml b/src/seclab_taskflows/taskflows/audit/fetch_security_advisories.yaml new file mode 100644 index 0000000..35b747b --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit/fetch_security_advisories.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" + +model_config: seclab_taskflows.configs.model_config + +globals: + repo: + +# Example taskflow to fetch and review security advisories for a repository +taskflow: + - task: + must_complete: true + exclude_from_context: false + agents: + - seclab_taskflow_agent.personalities.assistant + model: general_tasks + user_prompt: | + Fetch all GitHub Security Advisories (GHSAs) for the repo {{ globals.repo }}. + + If an error occurs during fetching, store the error message in memcache under the key 'security_advisories_{{ globals.repo }}'. + Ensure the error message starts with "Error:" followed by a description of the error. + + If fetching is successful, store the list of advisories in memcache under the key 'security_advisories_{{ globals.repo }}'. + + If one ore more advisories are found, provide a summary of the findings including: + 1. How many advisories were found + 2. The severity levels of the advisories + toolboxes: + - seclab_taskflows.toolboxes.ghsa + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.local_file_viewer + - seclab_taskflows.toolboxes.gh_file_viewer diff --git a/src/seclab_taskflows/taskflows/audit/filter_severity.yaml b/src/seclab_taskflows/taskflows/audit/filter_severity.yaml index ac91465..3e0a577 100644 --- a/src/seclab_taskflows/taskflows/audit/filter_severity.yaml +++ b/src/seclab_taskflows/taskflows/audit/filter_severity.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2025 GitHub +# SPDX-FileCopyrightText: GitHub, Inc. # SPDX-License-Identifier: MIT seclab-taskflow-agent: