diff --git a/.github/workflows/activate-stack.yml b/.github/workflows/activate-stack.yml index b5137a0fd..4a4b39faa 100644 --- a/.github/workflows/activate-stack.yml +++ b/.github/workflows/activate-stack.yml @@ -15,16 +15,15 @@ on: required: true type: string -permissions: - id-token: write - contents: read - actions: write - jobs: activate-stack: name: Activate ${{ inputs.stack_name }} for ${{ inputs.environment }} runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + contents: read + id-token: write + actions: write steps: - name: Git clone - ${{ github.ref }} diff --git a/.github/workflows/deploy-account-wide-infra.yml b/.github/workflows/deploy-account-wide-infra.yml index 2ce42409b..31fc37515 100644 --- a/.github/workflows/deploy-account-wide-infra.yml +++ b/.github/workflows/deploy-account-wide-infra.yml @@ -13,11 +13,6 @@ on: description: Branch to deploy required: true -permissions: - id-token: write - contents: read - actions: write - jobs: check-selected-environment: name: Check Workflow Env @@ -39,6 +34,10 @@ jobs: environment: ${{ inputs.environment }} needs: [check-selected-environment] runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + contents: read + id-token: write + actions: write steps: - name: Git clone - ${{ inputs.branch_name }} @@ -97,6 +96,10 @@ jobs: needs: [terraform-plan] runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + contents: read + id-token: write + actions: write steps: - name: Git clone - ${{ inputs.branch_name }} diff --git a/.github/workflows/persistent-environment.yml b/.github/workflows/persistent-environment.yml index fcc47fcdd..26152ed54 100644 --- a/.github/workflows/persistent-environment.yml +++ b/.github/workflows/persistent-environment.yml @@ -14,15 +14,14 @@ on: description: Branch to deploy required: true -permissions: - id-token: write - contents: read - actions: write - jobs: build: name: Build - ${{ inputs.branch_name }} runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + id-token: write + contents: read + actions: write steps: - name: Git clone - ${{ inputs.branch_name }} @@ -78,6 +77,10 @@ jobs: needs: [build] environment: ${{ inputs.environment }} runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + contents: read + id-token: write + actions: write steps: - name: Git clone - ${{ inputs.branch_name }} @@ -151,6 +154,10 @@ jobs: needs: [terraform-plan] runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + contents: read + id-token: write + actions: write steps: - name: Git clone - ${{ inputs.branch_name }} @@ -227,6 +234,10 @@ jobs: needs: [terraform-apply] runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + contents: read + id-token: write + actions: write steps: - name: Git clone - ${{ inputs.branch_name }} @@ -258,6 +269,9 @@ jobs: needs: [activate-stack] runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + contents: read + id-token: write steps: - name: Git clone - ${{ inputs.branch_name }} @@ -289,6 +303,9 @@ jobs: if: always() && ( needs.post-release-verify.result == 'failure' ) runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + contents: read + id-token: write steps: - name: Git clone - ${{ inputs.branch_name }} diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 76e27080f..056fafef4 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -9,13 +9,6 @@ concurrency: group: environment-${{ github.event.pull_request.number }} cancel-in-progress: false -permissions: - id-token: write - contents: read - actions: write - issues: write - pull-requests: write - jobs: set-environment-id: name: Set Environment ID @@ -48,6 +41,13 @@ jobs: name: Build Application runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: pull-request + permissions: + id-token: write + contents: read + actions: write + issues: write + pull-requests: write + steps: - name: Git Clone - ${{ github.event.pull_request.head.ref }} uses: actions/checkout@v4 @@ -110,6 +110,12 @@ jobs: runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: pull-request needs: [set-environment-id, build] + permissions: + id-token: write + contents: read + actions: write + issues: write + pull-requests: write steps: - name: Git Clone - ${{ github.event.pull_request.head.ref }} @@ -194,6 +200,9 @@ jobs: needs: [set-environment-id, deploy] environment: pull-request runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + id-token: write + contents: read steps: - name: Git Clone - ${{ github.event.pull_request.head.ref }} @@ -232,6 +241,10 @@ jobs: needs: [set-environment-id, integration-test] environment: pull-request runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + id-token: write + contents: read + steps: - name: Git Clone - ${{ github.event.pull_request.head.ref }} uses: actions/checkout@v4 @@ -266,6 +279,10 @@ jobs: needs: [set-environment-id, integration-test] environment: pull-request runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + id-token: write + contents: read + actions: write steps: - name: Git Clone - ${{ github.event.pull_request.head.ref }} diff --git a/.github/workflows/pr-env-destroy.yml b/.github/workflows/pr-env-destroy.yml index 8c9c76900..22eef9240 100644 --- a/.github/workflows/pr-env-destroy.yml +++ b/.github/workflows/pr-env-destroy.yml @@ -10,13 +10,6 @@ concurrency: group: environment-${{ github.event.pull_request.number }} cancel-in-progress: true -permissions: - id-token: write - contents: read - actions: write - issues: write - pull-requests: write - jobs: set-environment-id: name: Set Environment ID @@ -50,6 +43,11 @@ jobs: needs: [set-environment-id] environment: pull-request runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + id-token: write + contents: read + issues: write + pull-requests: write steps: - name: Git Clone - ${{ github.event.pull_request.head.ref }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa7dc4d8b..1fd1d28a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,5 @@ name: Release Published run-name: Release NRL ${{ github.event.release.name }} -permissions: - id-token: write - contents: write - actions: write env: SYFT_VERSION: "1.27.1" @@ -11,15 +7,16 @@ env: on: release: types: [published] - # push: - # tags: - # - v* workflow_dispatch: jobs: sbom: name: Generate Software Bill of Materials - ${{ github.event.release.name }} runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + id-token: write + contents: write + actions: write steps: - name: Git clone - ${{ github.ref }} diff --git a/.github/workflows/rollback-stack.yml b/.github/workflows/rollback-stack.yml index 2d2a31ae8..77590d082 100644 --- a/.github/workflows/rollback-stack.yml +++ b/.github/workflows/rollback-stack.yml @@ -10,16 +10,15 @@ on: default: "dev" type: environment -permissions: - id-token: write - contents: read - actions: write - jobs: rollback-stack: name: Rollback to inactive stack for ${{ inputs.environment }} runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + id-token: write + contents: read + actions: write steps: - name: Git clone - ${{ github.ref }} diff --git a/.github/workflows/update-lambda-permissions.yml b/.github/workflows/update-lambda-permissions.yml index 3228738a6..e3b7aaed5 100644 --- a/.github/workflows/update-lambda-permissions.yml +++ b/.github/workflows/update-lambda-permissions.yml @@ -21,15 +21,13 @@ on: type: boolean default: true -permissions: - id-token: write - contents: read - actions: write - jobs: check-versions: name: Check versions runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} + permissions: + id-token: write + contents: read steps: - name: Git clone - ${{ github.ref }} @@ -80,6 +78,10 @@ jobs: name: Build permissions runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} + permissions: + id-token: write + contents: read + actions: write needs: [check-versions] @@ -119,8 +121,11 @@ jobs: name: Pull deployed lambdas runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} - needs: [check-versions] + permissions: + id-token: write + contents: read + actions: write steps: - name: Git clone - ${{ github.ref }} @@ -161,8 +166,11 @@ jobs: name: Plan changes runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} - needs: [build-permissions, pull-deployed-lambdas] + permissions: + id-token: write + contents: read + actions: write steps: - name: Git clone - ${{ github.ref }} @@ -227,8 +235,11 @@ jobs: name: Apply permissions runs-on: codebuild-nhsd-nrlf-ci-build-project-${{ github.run_id }}-${{ github.run_attempt }} environment: ${{ inputs.environment }} - needs: terraform-plan + permissions: + id-token: write + contents: read + actions: read steps: - name: Git clone - ${{ github.ref }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6b52d3a9..71f55671b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: - flake8-print args: - "--select=T201,F401,F402,F403" - - "--exclude=.git,__pycache__,dist,.venv,scripts/*,packages/feature_documentation/*,layer/psycopg2/*,changelog/scripts/changelog.py" + - "--exclude=.git,__pycache__,dist,.venv,scripts/*" - repo: https://github.com/psf/black rev: 24.3.0 @@ -70,28 +70,3 @@ repos: args: - --args=-write=true - --args=-recursive - - # - repo: local - # hooks: - # - id: forbid_json_loads - # name: Don't use json.loads - use json_loads instead - # entry: json\.loads - # language: pygrep - # types: [python] - # exclude: layer/nrlf/nrlf/core/validators.py|layer/psycopg2/.*|mi/.* - - # - repo: local - # hooks: - # - id: forbid_json_load - # name: Don't use json.load - use json_load instead - # entry: json\.load - # language: pygrep - # types: [python] - # exclude: layer/nrlf/nrlf/core/validators.py|layer/psycopg2/.*|mi/.* - - - repo: local - hooks: - - id: create_changelog - name: Create changelog from changelog files - entry: changelog/scripts/changelog-pre-commit.sh - language: python diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index ad4b13435..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,211 +0,0 @@ -# Changelog - -## 2024-03-14 - -- NRL-511 - Capability statements -- NRL-496 - Update Format code system URL -- NRL-521 - Port (duplicate) POST functionality to PUT Interaction - -## 2024-02-08 - -- SPINECLI-1795 Add open source links to documentation - -## 2024-01-09 - -- NRLF-657 Add workspace destroy and unlock commands -- SPINECLI-1665 Add asid-check data contract for the new pointer type Lloyd george -- NRLF-708 Fix sonarcloud security hotspot changes -- SPINECLI-1704 add checks and cleanup to contract integration tests -- NRL-445 Add Allure reports -- SPINECLI-1666 Documentation Updates -- SPINECLI-1774 Fix integration test failures - -## 2023-10-26 - -- SPINECLI-1611 - Create sonar-project.properties file for SonarCloud configuration - -## 2023-10-09 - -- SPINECLI-1090 - Update NRLF Readme with WSL and Powershell Instructions -- SPINECLI-1619 - Change the readme - updated section on Reporting to add more info - -## 2023-09-21 - -- SPINECLI-1190 - Add link to postman collection in narrative -- NRLF-691 - Amend Document Type used in NRLF Smoke Test to clearly identify as Test-Only in MI Reports -- SPINECLI-1043 - 403 when x-correlation-id or x-request-id is missing - -## 2023-09-13 - -- NRLF-696 - BUG: MI report date range -- NRLF-697 - Enhancement: MI Max Year -- NRLF-702 - Add Postman Collection to API Catalogue - -## 2023-09-06 - -- NRLF-644 - Add missing "Errors" to swagger response in Consumer. -- NRLF-646 - Add missing "Errors" to swagger response in Producer. -- NRLF-663 - Add Open Source section to Producer/Consumer Narrative -- NRLF-693 - Sonarcube Fixes -- NRLF-682 - Remove documentation from CI -- NRLF-698 - Fix Forward: 4XX isn’t valid in API Gateway -- NRLF-582 - (APIGEE) 64L CORS issue with "TRY" Section in API Catalogue -- NRLF-690 - (APIGEE) Update dependency - -## 2023-08-29 - -- NRLF-563 - 64L Sort fields that should be treated as Sets, not ordered Lists -- NRLF-675 - MI dead letters, resubmission and alerting -- NRLF-683 - MI Documentation and reports -- NRLF-689 - Update NRLF 1D Converter to v0.0.9 -- Dependabot updates: - - pydantic (1.10.10 -> 1.10.12) - - boto3 (1.26.165 -> 1.28.30) - - cryptography (41.0.1 -> 41.0.3) - - awscli (1.27.165 -> 1.29.30) - - datamodel-code-generator (0.21.2 -> 0.21.4) - - cookiecutter (2.2.3 -> 2.3.0) - - atlassian-python-api (3.39.0 -> 3.41.0) - - hypothesis (6.82.0 -> 6.82.6) - - more-itertools (9.1.0 -> 10.1.0) - -## 2023-08-21 - -- NRLF-669 - Changelog now uses rendered markdown -- NRLF-653 - Producer MI - Create data model and insert into database -- NRLF-583 - Remove 'newman' from APIGEE proxies (APIGEE repos only) - -## 2023-08-16 - -- NRLF-561 - Fix forward: Add missing NEWS2 document type data contract - -## 2023-08-14 - -- NRLF-533 - 409 Conflict error not thrown on duplicate supersede request -- NRLF-585 - Add Live URLs to API Catalogue -- NRLF-652 - Producer MI - Add infrastructure for dynamodb streams -- NRLF-561 - Build an ASID specific document contract -- NRLF-634 - Helper script to create Data Contracts -- NRLF-642 - Update converter to v0.0.8 in NRLF repository -- NRLF-652 - Fix forward: Remove tags from security groups associations -- NRLF-561 - Fix forward: Fix ASID url -- NRLF-634 - Fix forward: Fix intermittent deploy contract test -- NRLF-494 - Fix forward: Revert changes introduced by NRLF-494, due to being too restrictive for converter - -## 2023-08-03a - -- NRLF-491 - Implement int-sandbox splunk environment -- NRLF-525 - Add versioning to firehose S3 bucket -- NRLF-529 - ASDF Tool manager -- NRLF-613 - Increase lambda memory size -- NRLF-638 - Can create documents with contact details -- NRLF-658 - Consumer S3 Auth - -## 2023-07-27 - -- NRLF-526 - CIS2 -- NRLF-560 - JSON Schema / Data contracts -- NRLF-640 - Add changelog details to narrative -- NRLF-650 - Dependabot updates -- NRLF-651 - Converter tests -- NRLF-655 - Tag/release fix - -## 2023-07-19 - -- NRLF-586 - Consolidate and publish CHANGELOG.md -- NRLF-460 - Merge steps for search, searchPost and count -- NRLF-599 - SSO -- NRLF-620 - Update the IAM-developer role for mgmt to allow them to call test secrets (done within NRLF-599 branch) -- NRLF-624 - Dependabot part 4 - More updates from the dependabot sorting and merging -- NRLF-627 - The release tag creation mechanism needs to use changelog file - -## 2023-07-11a - -- NRLF-605 - Update Readme -- NRLF-506 - [MI/BI] DB schema and querying the DB - -## 2023-07-06 - -- NRLF 559 - author is included in set of immutable fields -- NRLF 492 - Use latest change log for release tag -- NRLF 610 - dependabot changes -- NRLF 616 - Added feature test for fully populated NRL pointer - -## 2023-06-30 - -- Fix Count End Point has incorrect parameters (NRLF-577) -- Fix URL in prod smoke tests (NRLF-614) -- Create operation should fail when id(part after '-') has blank or special characters (NRLF-494) - -## 2023-06-29 - -- Remove double hyphen from RDS cluster name - -## 2023-06-23 - -- Authoriser parses headers for authorization into the "request context" to be used by API lambdas -- S3 bucket implemented for authorization lookup where nrl.enable-authorisation-lookup has been set -- All API lambdas read the authorisation from the event "request context" instead of the event headers -- (in nrl-producer-api repo) Update producer apigee proxy to optionally use enable-authorisation-lookup instead of nrl-ods-XXX - -## 2023-06-20 - -- NRLF-522 - errors section in narrative -- NRLF-584 - dependabot changes -- NRLF-564 - strip empty elements - -## 2023-06-14 - -- NRLF-505 - RDS instance, cluster and other terraform and lambda changes for MI/BI -- NRLF-517 - smoke tests that cover the NRL sync functionality -- NRLF-550 - Corrections to the ASID system value - -## 2023-06-13 - -- NRLF-578-dependabot changes - -## 2023-05-30 - -- NRLF-535 - New count endpoint -- NRLF-543 - type.code feature tests -- NRLF-424 - Remove ASID from swagger -- NRLF-482 - Make ID immutable field - -## 2023-05-15 - -- Optimisation of status lambda - -## 2023-05-11 - -- Changes to the onboarding documents -- Changed the duplicateError to return a 409 instead of 400 -- Syncing supersede date - -## 2023-05-10 - -- Supersede will not fail on delete for data sync permissions - -## 2023-05-03 - -- Firehose error alerts -- Superseding of created_on and when caller has permission audit-dates-from-payload - -## 2023-05-02 - -Security enhancements: - -- Deletion protection on DynamoDB -- Dependabot improvements - -## 2023-04-30 - -- Fix Count End Point has incorrect parameters -- Fix URL in prod smoke tests -- Create operation should fail when id(part after '-') has blank or special characters - -## 2023-04-25 - -- Multiple End-User organisations support -- Logging improvements -- Build improvements -- Authentication improvements diff --git a/api/consumer/searchDocumentReference/search_document_reference.py b/api/consumer/searchDocumentReference/search_document_reference.py index 58b55af37..d87fbebb0 100644 --- a/api/consumer/searchDocumentReference/search_document_reference.py +++ b/api/consumer/searchDocumentReference/search_document_reference.py @@ -66,7 +66,7 @@ def handler( logger.log( LogReference.CONSEARCH002b, category=params.category, - ) # TODO - Should update error message once permissioning by category is implemented + ) return SpineErrorResponse.INVALID_CODE_SYSTEM( diagnostics="Invalid query parameter (The provided category is not valid)", expression="category", diff --git a/api/consumer/searchPostDocumentReference/search_post_document_reference.py b/api/consumer/searchPostDocumentReference/search_post_document_reference.py index fd970111a..44bd33e61 100644 --- a/api/consumer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/consumer/searchPostDocumentReference/search_post_document_reference.py @@ -66,7 +66,7 @@ def handler( logger.log( LogReference.CONPOSTSEARCH002b, category=body.category, - ) # TODO - Should update error message once permissioning by category is implemented + ) return SpineErrorResponse.INVALID_CODE_SYSTEM( diagnostics="The provided category is not valid", expression="category", diff --git a/api/producer/searchDocumentReference/search_document_reference.py b/api/producer/searchDocumentReference/search_document_reference.py index d201f8abf..1641d6adb 100644 --- a/api/producer/searchDocumentReference/search_document_reference.py +++ b/api/producer/searchDocumentReference/search_document_reference.py @@ -68,7 +68,7 @@ def handler( logger.log( LogReference.PROSEARCH002b, category=params.category, - ) # TODO - Should update error message once permissioning by category is implemented + ) return SpineErrorResponse.INVALID_CODE_SYSTEM( diagnostics="Invalid query parameter (The provided category is not valid)", expression="category", diff --git a/api/producer/searchPostDocumentReference/search_post_document_reference.py b/api/producer/searchPostDocumentReference/search_post_document_reference.py index cac1ce30d..f8a5bd250 100644 --- a/api/producer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/producer/searchPostDocumentReference/search_post_document_reference.py @@ -62,7 +62,7 @@ def handler( logger.log( LogReference.PROPOSTSEARCH002b, category=body.category, - ) # TODO - Should update error message once permissioning by category is implemented + ) return SpineErrorResponse.INVALID_CODE_SYSTEM( diagnostics="The provided category is not valid", expression="category", diff --git a/changelog/2023-04-25.md b/changelog/2023-04-25.md deleted file mode 100644 index 0bcefee06..000000000 --- a/changelog/2023-04-25.md +++ /dev/null @@ -1,4 +0,0 @@ -- Multiple End-User organisations support -- Logging improvements -- Build improvements -- Authentication improvements diff --git a/changelog/2023-04-30.md b/changelog/2023-04-30.md deleted file mode 100644 index 8808d1306..000000000 --- a/changelog/2023-04-30.md +++ /dev/null @@ -1,3 +0,0 @@ -- Fix Count End Point has incorrect parameters -- Fix URL in prod smoke tests -- Create operation should fail when id(part after '-') has blank or special characters diff --git a/changelog/2023-05-02.md b/changelog/2023-05-02.md deleted file mode 100644 index 7670df01f..000000000 --- a/changelog/2023-05-02.md +++ /dev/null @@ -1,4 +0,0 @@ -Security enhancements: - -- Deletion protection on DynamoDB -- Dependabot improvements diff --git a/changelog/2023-05-03.md b/changelog/2023-05-03.md deleted file mode 100644 index af8dc3334..000000000 --- a/changelog/2023-05-03.md +++ /dev/null @@ -1,2 +0,0 @@ -- Firehose error alerts -- Superseding of created_on and when caller has permission audit-dates-from-payload diff --git a/changelog/2023-05-10.md b/changelog/2023-05-10.md deleted file mode 100644 index 4e7887daa..000000000 --- a/changelog/2023-05-10.md +++ /dev/null @@ -1 +0,0 @@ -- Supersede will not fail on delete for data sync permissions diff --git a/changelog/2023-05-11.md b/changelog/2023-05-11.md deleted file mode 100644 index 01fa3fa37..000000000 --- a/changelog/2023-05-11.md +++ /dev/null @@ -1,3 +0,0 @@ -- Changes to the onboarding documents -- Changed the duplicateError to return a 409 instead of 400 -- Syncing supersede date diff --git a/changelog/2023-05-15.md b/changelog/2023-05-15.md deleted file mode 100644 index 02a422362..000000000 --- a/changelog/2023-05-15.md +++ /dev/null @@ -1 +0,0 @@ -- Optimisation of status lambda diff --git a/changelog/2023-05-30.md b/changelog/2023-05-30.md deleted file mode 100644 index 4c403387c..000000000 --- a/changelog/2023-05-30.md +++ /dev/null @@ -1,4 +0,0 @@ -- NRLF-535 - New count endpoint -- NRLF-543 - type.code feature tests -- NRLF-424 - Remove ASID from swagger -- NRLF-482 - Make ID immutable field diff --git a/changelog/2023-06-13.md b/changelog/2023-06-13.md deleted file mode 100644 index acd444aae..000000000 --- a/changelog/2023-06-13.md +++ /dev/null @@ -1 +0,0 @@ -- NRLF-578-dependabot changes diff --git a/changelog/2023-06-14.md b/changelog/2023-06-14.md deleted file mode 100644 index 891f8881e..000000000 --- a/changelog/2023-06-14.md +++ /dev/null @@ -1,3 +0,0 @@ -- NRLF-505 - RDS instance, cluster and other terraform and lambda changes for MI/BI -- NRLF-517 - smoke tests that cover the NRL sync functionality -- NRLF-550 - Corrections to the ASID system value diff --git a/changelog/2023-06-20.md b/changelog/2023-06-20.md deleted file mode 100644 index cce05eee0..000000000 --- a/changelog/2023-06-20.md +++ /dev/null @@ -1,3 +0,0 @@ -- NRLF-522 - errors section in narrative -- NRLF-584 - dependabot changes -- NRLF-564 - strip empty elements diff --git a/changelog/2023-06-23.md b/changelog/2023-06-23.md deleted file mode 100644 index a20d314d0..000000000 --- a/changelog/2023-06-23.md +++ /dev/null @@ -1,4 +0,0 @@ -- Authoriser parses headers for authorization into the "request context" to be used by API lambdas -- S3 bucket implemented for authorization lookup where nrl.enable-authorisation-lookup has been set -- All API lambdas read the authorisation from the event "request context" instead of the event headers -- (in nrl-producer-api repo) Update producer apigee proxy to optionally use enable-authorisation-lookup instead of nrl-ods-XXX diff --git a/changelog/2023-06-29.md b/changelog/2023-06-29.md deleted file mode 100644 index 269a8099b..000000000 --- a/changelog/2023-06-29.md +++ /dev/null @@ -1 +0,0 @@ -- Remove double hyphen from RDS cluster name diff --git a/changelog/2023-06-30.md b/changelog/2023-06-30.md deleted file mode 100644 index 93eb19759..000000000 --- a/changelog/2023-06-30.md +++ /dev/null @@ -1,3 +0,0 @@ -- Fix Count End Point has incorrect parameters (NRLF-577) -- Fix URL in prod smoke tests (NRLF-614) -- Create operation should fail when id(part after '-') has blank or special characters (NRLF-494) diff --git a/changelog/2023-07-06.md b/changelog/2023-07-06.md deleted file mode 100644 index 0787c9cb1..000000000 --- a/changelog/2023-07-06.md +++ /dev/null @@ -1,4 +0,0 @@ -- NRLF 559 - author is included in set of immutable fields -- NRLF 492 - Use latest change log for release tag -- NRLF 610 - dependabot changes -- NRLF 616 - Added feature test for fully populated NRL pointer diff --git a/changelog/2023-07-11a.md b/changelog/2023-07-11a.md deleted file mode 100644 index 97ca4585f..000000000 --- a/changelog/2023-07-11a.md +++ /dev/null @@ -1,2 +0,0 @@ -- NRLF-605 - Update Readme -- NRLF-506 - [MI/BI] DB schema and querying the DB diff --git a/changelog/2023-07-19.md b/changelog/2023-07-19.md deleted file mode 100644 index a2e0dbc46..000000000 --- a/changelog/2023-07-19.md +++ /dev/null @@ -1,6 +0,0 @@ -- NRLF-586 - Consolidate and publish CHANGELOG.md -- NRLF-460 - Merge steps for search, searchPost and count -- NRLF-599 - SSO -- NRLF-620 - Update the IAM-developer role for mgmt to allow them to call test secrets (done within NRLF-599 branch) -- NRLF-624 - Dependabot part 4 - More updates from the dependabot sorting and merging -- NRLF-627 - The release tag creation mechanism needs to use changelog file diff --git a/changelog/2023-07-27.md b/changelog/2023-07-27.md deleted file mode 100644 index 68b4feeaf..000000000 --- a/changelog/2023-07-27.md +++ /dev/null @@ -1,6 +0,0 @@ -- NRLF-526 - CIS2 -- NRLF-560 - JSON Schema / Data contracts -- NRLF-640 - Add changelog details to narrative -- NRLF-650 - Dependabot updates -- NRLF-651 - Converter tests -- NRLF-655 - Tag/release fix diff --git a/changelog/2023-08-03a.md b/changelog/2023-08-03a.md deleted file mode 100644 index 2a2c0c4f1..000000000 --- a/changelog/2023-08-03a.md +++ /dev/null @@ -1,6 +0,0 @@ -- NRLF-491 - Implement int-sandbox splunk environment -- NRLF-525 - Add versioning to firehose S3 bucket -- NRLF-529 - ASDF Tool manager -- NRLF-613 - Increase lambda memory size -- NRLF-638 - Can create documents with contact details -- NRLF-658 - Consumer S3 Auth diff --git a/changelog/2023-08-14.md b/changelog/2023-08-14.md deleted file mode 100644 index adafb03a5..000000000 --- a/changelog/2023-08-14.md +++ /dev/null @@ -1,10 +0,0 @@ -- NRLF-533 - 409 Conflict error not thrown on duplicate supersede request -- NRLF-585 - Add Live URLs to API Catalogue -- NRLF-652 - Producer MI - Add infrastructure for dynamodb streams -- NRLF-561 - Build an ASID specific document contract -- NRLF-634 - Helper script to create Data Contracts -- NRLF-642 - Update converter to v0.0.8 in NRLF repository -- NRLF-652 - Fix forward: Remove tags from security groups associations -- NRLF-561 - Fix forward: Fix ASID url -- NRLF-634 - Fix forward: Fix intermittent deploy contract test -- NRLF-494 - Fix forward: Revert changes introduced by NRLF-494, due to being too restrictive for converter diff --git a/changelog/2023-08-16.md b/changelog/2023-08-16.md deleted file mode 100644 index c05322a04..000000000 --- a/changelog/2023-08-16.md +++ /dev/null @@ -1 +0,0 @@ -- NRLF-561 - Fix forward: Add missing NEWS2 document type data contract diff --git a/changelog/2023-08-21.md b/changelog/2023-08-21.md deleted file mode 100644 index 2de10e955..000000000 --- a/changelog/2023-08-21.md +++ /dev/null @@ -1,3 +0,0 @@ -- NRLF-669 - Changelog now uses rendered markdown -- NRLF-653 - Producer MI - Create data model and insert into database -- NRLF-583 - Remove 'newman' from APIGEE proxies (APIGEE repos only) diff --git a/changelog/2023-08-29.md b/changelog/2023-08-29.md deleted file mode 100644 index faa968029..000000000 --- a/changelog/2023-08-29.md +++ /dev/null @@ -1,14 +0,0 @@ -- NRLF-563 - 64L Sort fields that should be treated as Sets, not ordered Lists -- NRLF-675 - MI dead letters, resubmission and alerting -- NRLF-683 - MI Documentation and reports -- NRLF-689 - Update NRLF 1D Converter to v0.0.9 -- Dependabot updates: - - pydantic (1.10.10 -> 1.10.12) - - boto3 (1.26.165 -> 1.28.30) - - cryptography (41.0.1 -> 41.0.3) - - awscli (1.27.165 -> 1.29.30) - - datamodel-code-generator (0.21.2 -> 0.21.4) - - cookiecutter (2.2.3 -> 2.3.0) - - atlassian-python-api (3.39.0 -> 3.41.0) - - hypothesis (6.82.0 -> 6.82.6) - - more-itertools (9.1.0 -> 10.1.0) diff --git a/changelog/2023-09-06.md b/changelog/2023-09-06.md deleted file mode 100644 index 618c4c277..000000000 --- a/changelog/2023-09-06.md +++ /dev/null @@ -1,8 +0,0 @@ -- NRLF-644 - Add missing "Errors" to swagger response in Consumer. -- NRLF-646 - Add missing "Errors" to swagger response in Producer. -- NRLF-663 - Add Open Source section to Producer/Consumer Narrative -- NRLF-693 - Sonarcube Fixes -- NRLF-682 - Remove documentation from CI -- NRLF-698 - Fix Forward: 4XX isn’t valid in API Gateway -- NRLF-582 - (APIGEE) 64L CORS issue with "TRY" Section in API Catalogue -- NRLF-690 - (APIGEE) Update dependency diff --git a/changelog/2023-09-13.md b/changelog/2023-09-13.md deleted file mode 100644 index 9d615bf85..000000000 --- a/changelog/2023-09-13.md +++ /dev/null @@ -1,3 +0,0 @@ -- NRLF-696 - BUG: MI report date range -- NRLF-697 - Enhancement: MI Max Year -- NRLF-702 - Add Postman Collection to API Catalogue diff --git a/changelog/2023-09-21.md b/changelog/2023-09-21.md deleted file mode 100644 index 168f47d3f..000000000 --- a/changelog/2023-09-21.md +++ /dev/null @@ -1,3 +0,0 @@ -- SPINECLI-1190 - Add link to postman collection in narrative -- NRLF-691 - Amend Document Type used in NRLF Smoke Test to clearly identify as Test-Only in MI Reports -- SPINECLI-1043 - 403 when x-correlation-id or x-request-id is missing diff --git a/changelog/2023-10-09.md b/changelog/2023-10-09.md deleted file mode 100644 index beb3860d4..000000000 --- a/changelog/2023-10-09.md +++ /dev/null @@ -1,2 +0,0 @@ -- SPINECLI-1090 - Update NRLF Readme with WSL and Powershell Instructions -- SPINECLI-1619 - Change the readme - updated section on Reporting to add more info diff --git a/changelog/2023-10-26.md b/changelog/2023-10-26.md deleted file mode 100644 index 948cceca5..000000000 --- a/changelog/2023-10-26.md +++ /dev/null @@ -1 +0,0 @@ -- SPINECLI-1611 - Create sonar-project.properties file for SonarCloud configuration diff --git a/changelog/2024-01-09.md b/changelog/2024-01-09.md deleted file mode 100644 index 4094a6266..000000000 --- a/changelog/2024-01-09.md +++ /dev/null @@ -1,7 +0,0 @@ -- NRLF-657 Add workspace destroy and unlock commands -- SPINECLI-1665 Add asid-check data contract for the new pointer type Lloyd george -- NRLF-708 Fix sonarcloud security hotspot changes -- SPINECLI-1704 add checks and cleanup to contract integration tests -- NRL-445 Add Allure reports -- SPINECLI-1666 Documentation Updates -- SPINECLI-1774 Fix integration test failures diff --git a/changelog/2024-02-08.md b/changelog/2024-02-08.md deleted file mode 100644 index 5097a6c98..000000000 --- a/changelog/2024-02-08.md +++ /dev/null @@ -1 +0,0 @@ -- SPINECLI-1795 Add open source links to documentation diff --git a/changelog/2024-03-14.md b/changelog/2024-03-14.md deleted file mode 100644 index aeb40443b..000000000 --- a/changelog/2024-03-14.md +++ /dev/null @@ -1,3 +0,0 @@ -- NRL-511 - Capability statements -- NRL-496 - Update Format code system URL -- NRL-521 - Port (duplicate) POST functionality to PUT Interaction diff --git a/changelog/scripts/changelog-pre-commit.sh b/changelog/scripts/changelog-pre-commit.sh deleted file mode 100755 index 175820b24..000000000 --- a/changelog/scripts/changelog-pre-commit.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -branch=$(git symbolic-ref --short HEAD) -if [[ "${branch}" == *"release/"* ]]; then - result=`python ./changelog/scripts/changelog.py $branch` - if [[ "${result}" == "CHANGELOG.md updated." ]]; then - echo "SUCCESS: CHANGELOG.md was out of date and has now been updated. Please re-commit." - exit 1 - elif [[ "${result}" == "No matching changelog for release branch." ]]; then - echo "FAILURE: You must create a change log to match the release branch." - exit 1 - else - exit 0 - fi -else - exit 0 -fi diff --git a/changelog/scripts/changelog.py b/changelog/scripts/changelog.py deleted file mode 100644 index 27a49d2a1..000000000 --- a/changelog/scripts/changelog.py +++ /dev/null @@ -1,48 +0,0 @@ -import sys -from os import listdir -from os.path import getsize, isfile, join - - -def main(argv): - if "release/" in argv[0]: - file_list = sorted( - [f for f in listdir("./changelog") if isfile(join("./changelog", f))] - ) - file_list.reverse() - latest_file = file_list[0].replace(".md", "") - release_date = argv[0].replace("release/", "") - if latest_file != release_date: - print("No matching changelog for release branch.") - sys.exit(0) - else: - data = "# Changelog\n\n" - - for i, v in enumerate(file_list): - data = data + "## " + v.replace(".md", "") + "\n\n" - with open(f"./changelog/{v}", "r") as f: - data = data + f.read() - if i != (len(file_list) - 1): - data = data + "\n" - - try: - original_size = getsize("./CHANGELOG.md") - except OSError: - original_size = 0 - - with open("./CHANGELOG.md", "w") as f: - f.write(data) - new_size = getsize("./CHANGELOG.md") - - if new_size != original_size: - print("CHANGELOG.md updated.") - sys.exit(0) - - print("CHANGELOG.md, No changes.") - sys.exit(0) - - print("Not a release branch") - sys.exit(0) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/layer/nrlf/core/constants.py b/layer/nrlf/core/constants.py index 793c2458c..811c02266 100644 --- a/layer/nrlf/core/constants.py +++ b/layer/nrlf/core/constants.py @@ -72,7 +72,7 @@ class PointerTypes(Enum): @staticmethod def list(): - return list(map(lambda type: type.value, PointerTypes)) + return [type.value for type in PointerTypes] def coding_system(self): return self.value.split(TYPE_SEPARATOR)[0] @@ -92,7 +92,7 @@ class Categories(Enum): @staticmethod def list(): - return list(map(lambda category: category.value, Categories)) + return [category.value for category in Categories] def coding_system(self): return self.value.split(TYPE_SEPARATOR)[0] diff --git a/layer/nrlf/core/dynamodb/model.py b/layer/nrlf/core/dynamodb/model.py index c66bce77f..6f3ed4202 100644 --- a/layer/nrlf/core/dynamodb/model.py +++ b/layer/nrlf/core/dynamodb/model.py @@ -199,7 +199,7 @@ def validate_id(cls, id_: str) -> str: The id should be in the format - """ if not re.match( - r"^(?=.{1,64}$)[A-Za-z0-9|\\.]+-[A-Za-z0-9]+[A-Za-z0-9\\_\\-]*$", id_ + r"^(?=.{1,64}$)[A-Za-z0-9|\\.]+-[A-Za-z0-9]+[A-Za-z0-9_\\-]*$", id_ ): raise ValueError("id must be in the format -") @@ -230,7 +230,7 @@ def validate_type(cls, type: str) -> str: Validate the type of the DocumentPointer The type should be in the format | """ - if not re.match(r"^[A-Za-z0-9]+|[A-Za-z0-9]+$", type): + if not re.match(r"^[A-Za-z0-9:/\.]+\|[A-Za-z0-9]+$", type): raise ValueError("type must be in the format |") return type diff --git a/layer/nrlf/core/dynamodb/repository.py b/layer/nrlf/core/dynamodb/repository.py index 1f8262ff0..26a36deef 100644 --- a/layer/nrlf/core/dynamodb/repository.py +++ b/layer/nrlf/core/dynamodb/repository.py @@ -1,6 +1,5 @@ import sys -from abc import ABC -from typing import Generic, Iterator, List, Optional, Type, TypeVar +from typing import Iterator, List, Optional, Type from botocore.exceptions import ClientError from pydantic import ValidationError @@ -8,12 +7,10 @@ from nrlf.core.boto import get_dynamodb_resource, get_dynamodb_table from nrlf.core.codes import SpineErrorConcept from nrlf.core.constants import SYSTEM_SHORT_IDS, TYPE_CATEGORIES -from nrlf.core.dynamodb.model import DocumentPointer, DynamoDBModel +from nrlf.core.dynamodb.model import DocumentPointer, DynamoDBModel # noqa: F401 from nrlf.core.errors import OperationOutcomeError from nrlf.core.logger import LogReference, logger -RepositoryModel = TypeVar("RepositoryModel", bound=DynamoDBModel) - def _get_sk_ids_for_type(pointer_type: str) -> tuple: if pointer_type not in TYPE_CATEGORIES: @@ -33,8 +30,8 @@ def _get_sk_ids_for_type(pointer_type: str) -> tuple: return category_id, type_id -class Repository(ABC, Generic[RepositoryModel]): - ITEM_TYPE: Type[RepositoryModel] +class Repository[T: DynamoDBModel]: + ITEM_TYPE: Type[T] def __init__(self, table_name: str): self.dynamodb = get_dynamodb_resource() diff --git a/layer/nrlf/core/dynamodb/tests/test_model.py b/layer/nrlf/core/dynamodb/tests/test_model.py index c2d5cfc4f..e5eeea0a0 100644 --- a/layer/nrlf/core/dynamodb/tests/test_model.py +++ b/layer/nrlf/core/dynamodb/tests/test_model.py @@ -2,6 +2,7 @@ import pytest from freezegun import freeze_time +from pydantic import ValidationError from nrlf.core.constants import PointerTypes from nrlf.core.dynamodb.model import DocumentPointer, DynamoDBModel @@ -155,6 +156,16 @@ def test_document_pointer_from_document_reference_invalid(): assert str(error.value) == "'NoneType' object has no attribute 'coding'" +def test_document_pointer_from_document_reference_invalid_type_code(): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.type.coding[0].code = "INVALID_CODE" + + with pytest.raises(ValidationError) as error: + DocumentPointer.from_document_reference(doc_ref) + + assert "type must be in the format |" in str(error.value) + + def test_document_pointer_inject_producer_id(): values = {"id": "X26-999999-999999-99999999", "producer_id": None} diff --git a/layer/nrlf/core/model.py b/layer/nrlf/core/model.py index 7163d339f..3729305b9 100644 --- a/layer/nrlf/core/model.py +++ b/layer/nrlf/core/model.py @@ -1,5 +1,3 @@ -from typing import Union - from nhs_number import is_valid as is_valid_nhs_number from pydantic import BaseModel, Field, StrictStr @@ -9,7 +7,7 @@ class _NhsNumberMixin: @property - def nhs_number(self) -> Union[str, None]: + def nhs_number(self) -> str | None: if self.subject_identifier is None: return None diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index bac27ca9f..0f1c1a81f 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -146,7 +146,7 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_content(resource) self._validate_content_format(resource) self._validate_content_extension(resource) - self._validate_practiceSetting(resource) + self._validate_practice_setting(resource) except StopValidationError: logger.log(LogReference.VALIDATOR003) @@ -287,7 +287,6 @@ def _validate_asid(self, asid_references: list): diagnostics=f"Invalid ASID value '{asid_value}'. A single ASID consisting of 12 digits can be provided in the context.related field.", field=f"context.related[{idx}].identifier.value", ) - return def _validate_ssp_asid(self, model: DocumentReference): """ @@ -295,11 +294,9 @@ def _validate_ssp_asid(self, model: DocumentReference): """ ssp_content = any( - [ - content - for content in model.content - if content.attachment.url.startswith("ssp://") - ] + content + for content in model.content + if content.attachment.url.startswith("ssp://") ) logger.log(LogReference.VALIDATOR001, step="ssp_content_and_asid_exists") @@ -338,7 +335,6 @@ def _validate_ssp_asid(self, model: DocumentReference): diagnostics="Missing ASID identifier. context.related must contain a single valid ASID identifier when content contains an SSP URL", field="context.related", ) - return def _validate_type(self, model: DocumentReference): """ @@ -395,7 +391,7 @@ def _validate_category(self, model: DocumentReference): issue_code="business-rule", error_code="UNPROCESSABLE_ENTITY", diagnostics=f"Invalid category length: {len(model.category)} Category must only contain a single value", - field=f"category", + field="category", ) return @@ -406,7 +402,7 @@ def _validate_category(self, model: DocumentReference): issue_code="business-rule", error_code="UNPROCESSABLE_ENTITY", diagnostics=f"Invalid category coding length: {len(model.category[0].coding)} Category Coding must only contain a single value", - field=f"category[0].coding", + field="category[0].coding", ) return @@ -652,9 +648,8 @@ def _validate_author(self, model: DocumentReference): diagnostics=f"Invalid author value: '{identifier.value}' Author value must be less than 13 characters", field="author[0].identifier.value", ) - return - def _validate_practiceSetting(self, model: DocumentReference): + def _validate_practice_setting(self, model: DocumentReference): """ Validate the practice setting field contains an appropriate coding system and code. """ @@ -716,7 +711,6 @@ def _validate_practiceSetting(self, model: DocumentReference): diagnostics=f"Invalid practice setting coding: display {practice_setting_display} does not match the expected display for {practice_setting_value} Practice Setting coding is bound to value set {PRACTICE_SETTING_VALUE_SET_URL}", field="context.practiceSetting.coding[0]", ) - return def _validate_content(self, model: DocumentReference): """ diff --git a/scripts/aws_session_assume.py b/scripts/aws_session_assume.py index bec8fa8e7..081b36fca 100644 --- a/scripts/aws_session_assume.py +++ b/scripts/aws_session_assume.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +import os + import boto3 _AWS_ACCOUNT_FOR_ENV = { @@ -13,6 +15,8 @@ "prod": "prod", } +AWS_REGION = os.getenv("AWS_REGION", "eu-west-2") + def get_account_name(env: str): if env not in _AWS_ACCOUNT_FOR_ENV: @@ -23,7 +27,7 @@ def get_account_name(env: str): def get_account_id(env: str): account_name = get_account_name(env) - secretsmanager = boto3.client("secretsmanager", region_name="eu-west-2") + secretsmanager = boto3.client("secretsmanager", region_name=AWS_REGION) secret_id = f"nhsd-nrlf--mgmt--{account_name}-account-id" result = secretsmanager.get_secret_value(SecretId=secret_id) account_id = result["SecretString"] @@ -34,7 +38,7 @@ def get_account_id(env: str): def get_boto_session(env: str) -> boto3.Session: account_id = get_account_id(env) - sts = boto3.client("sts", region_name="eu-west-2") + sts = boto3.client("sts", region_name=AWS_REGION) result = sts.assume_role( RoleArn=f"arn:aws:iam::{account_id}:role/terraform", RoleSessionName="get-account-id", diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 20280d768..76d66a54c 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -28,10 +28,11 @@ function _bootstrap_help() { } function _check_mgmt() { - if [ "$(aws iam list-account-aliases --query 'AccountAliases[0]' --output text)" != "nhsd-ddc-spine-nrlf-mgmt" ]; then + if [[ "$(aws iam list-account-aliases --query 'AccountAliases[0]' --output text)" != "nhsd-ddc-spine-nrlf-mgmt" ]]; then echo "Please log in as the mgmt account" >&2 return 1 fi + return 0 } function _check_non_mgmt() { @@ -39,6 +40,7 @@ function _check_non_mgmt() { echo "Please log in as a non-mgmt account" >&2 return 1 fi + return 0 } function _bootstrap() { @@ -98,7 +100,7 @@ function _bootstrap() { set +e mgmt_account_id=$(aws secretsmanager get-secret-value --secret-id "${MGMT_ACCOUNT_ID_LOCATION}" --query SecretString --output text) - if [ "${mgmt_account_id}" == "" ]; then + if [[ "${mgmt_account_id}" == "" ]]; then aws secretsmanager create-secret --name "${MGMT_ACCOUNT_ID_LOCATION}" echo "Please set ${MGMT_ACCOUNT_ID_LOCATION} in the Secrets Manager and rerun the script" exit 1 @@ -120,16 +122,10 @@ function _bootstrap() { #---------------- "destroy-non-mgmt") _check_non_mgmt || return 1 - # TODO: Reintroduce the admin check - but should be fine for all developers - # if [[ "$(aws sts get-caller-identity)" != *dev* || "$(aws sts get-caller-identity)" != *NHSDAdminRole* ]]; then - # echo "Please log in as dev with an Admin account" >&2 - # return 1 - # fi - local workspace workspace=$2 # Fetch the resources using the AWS CLI command - aws resourcegroupstaggingapi get-resources --tag-filters Key=workspace,Values="$2" | jq -c '.ResourceTagMappingList[]' | + aws resourcegroupstaggingapi get-resources --tag-filters Key=workspace,Values="${workspace}" | jq -c '.ResourceTagMappingList[]' | while IFS= read -r item; do arn=$(jq -r '.ResourceARN' <<< "$item") @@ -146,7 +142,7 @@ function _bootstrap() { ;; arn:aws:logs* ) echo "Deleting... : $arn" - new_var=$(echo "$arn" | awk -F':' '{print $NF}') + new_var=$(echo "$arn" | awk -F':' '{print $NF}') # NOSONAR (S1192) NF is not a env var aws logs delete-log-group --log-group-name $new_var ;; arn:aws:secretsmanager* ) @@ -162,13 +158,13 @@ function _bootstrap() { ;; arn:aws:dynamodb* ) echo "Deleting... : $arn" - new_var=$(echo "$arn" | awk -F':' '{print $NF}') - table=$(echo "$arn" | awk -F'/' '{print $NF}') + new_var=$(echo "$arn" | awk -F':' '{print $NF}') # NOSONAR (S1192) NF is not a env var + table=$(echo "$arn" | awk -F'/' '{print $NF}') # NOSONAR (S1192) NF is not a env var aws dynamodb delete-table --table-name $table ;; arn:aws:s3* ) echo "Deleting... : $arn" - new_var=$(echo "$arn" | awk -F':' '{print $NF}') + new_var=$(echo "$arn" | awk -F':' '{print $NF}') # NOSONAR (S1192) NF is not a env var local versioned_objects versioned_objects=$(aws s3api list-object-versions \ --bucket "${new_var}" \ @@ -182,9 +178,9 @@ function _bootstrap() { ;; arn:aws:ssm* ) echo "Deleting... : $arn" - new_var=$(echo "$arn" | awk -F':' '{print $NF}') - suffix=$(echo "$arn" | awk -F'/' '{print $NF}') - name=$(echo "$new_var" | awk -F'/' '{print $(NF-1)}') + new_var=$(echo "$arn" | awk -F':' '{print $NF}') # NOSONAR (S1192) NF is not a env var + suffix=$(echo "$arn" | awk -F'/' '{print $NF}') # NOSONAR (S1192) NF is not a env var + name=$(echo "$new_var" | awk -F'/' '{print $(NF-1)}') # NOSONAR (S1192) NF is not a env var aws ssm delete-parameter --name $name/$suffix ;; arn:aws:acm* ) @@ -193,8 +189,8 @@ function _bootstrap() { ;; arn:aws:firehose* ) echo "Deleting... : $arn" - new_var=$(echo "$arn" | awk -F':' '{print $NF}') - name=$(echo "$new_var" | awk -F'/' '{print $NF}') + new_var=$(echo "$arn" | awk -F':' '{print $NF}') # NOSONAR (S1192) NF is not a env var + name=$(echo "$new_var" | awk -F'/' '{print $NF}') # NOSONAR (S1192) NF is not a env var aws firehose delete-delivery-stream --delivery-stream-name $name ;; * ) @@ -206,6 +202,8 @@ function _bootstrap() { #---------------- *) _bootstrap_help ;; esac + + return 0 } _bootstrap "${@:1}" diff --git a/scripts/check-build-environment.sh b/scripts/check-build-environment.sh index 59fd95018..e1edce634 100755 --- a/scripts/check-build-environment.sh +++ b/scripts/check-build-environment.sh @@ -6,15 +6,17 @@ set -o errexit -o pipefail -o nounset : "${SHOULD_WARN_ONLY:="false"}" function success() { - [ "${SHOULD_WARN_ONLY}" == "true" ] && return + [[ "${SHOULD_WARN_ONLY}" == "true" ]] && return local message="$1" echo " ✅ ${message}" + return 0 } function warning() { local message="$1" echo -e " ⚠️ \e[31m${message}\e[39m" + return 0 } echo @@ -38,7 +40,7 @@ for dep in ${BUILD_DEPENDENCIES}; do dep_path="$(which ${dep} 2> /dev/null)" set -e - if [ -n "${dep_path}" -a -x "${dep_path}" ] + if [[ -n "${dep_path}" && -x "${dep_path}" ]] then success "${dep} found at ${dep_path}" else diff --git a/scripts/check-deploy-environment.sh b/scripts/check-deploy-environment.sh index f38552dcd..ec5b9b050 100755 --- a/scripts/check-deploy-environment.sh +++ b/scripts/check-deploy-environment.sh @@ -9,15 +9,17 @@ set -o errexit -o pipefail -o nounset : "${TF_WORKSPACE_NAME:=""}" function success() { - [ "${SHOULD_WARN_ONLY}" == "true" ] && return + [[ "${SHOULD_WARN_ONLY}" == "true" ]] && return local message="$1" echo " ✅ ${message}" + return 0 } function warning() { local message="$1" echo -e " ⚠️ \e[31m${message}\e[39m" + return 0 } echo @@ -33,7 +35,7 @@ for dep in ${DEPLOY_DEPENDENCIES}; do dep_path="$(which ${dep} 2> /dev/null)" set -e - if [ -n "${dep_path}" -a -x "${dep_path}" ] + if [[ -n "${dep_path}" -a -x "${dep_path}" ]] then success "${dep} found at ${dep_path}" else @@ -45,7 +47,7 @@ done set +e env_account_id="$(aws secretsmanager get-secret-value --secret-id nhsd-nrlf--mgmt--${ENV_ACCOUNT_NAME}-account-id --query SecretString --output text)" set -e -if [ -n "${env_account_id}" ] +if [[ -n "${env_account_id}" ]] then success "${ENV_ACCOUNT_NAME} account id found in mgmt account" else @@ -59,7 +61,7 @@ tf_workspace="$(cd terraform/infrastructure && terraform workspace show)" set -e is_using_shared_resources="$(poetry run python ./scripts/are_resources_shared_for_stack.py ${tf_workspace})" -if [ "${is_using_shared_resources}" == "true" ] +if [[ "${is_using_shared_resources}" == "true" ]] then warning "Will use shared resources for stack '${tf_workspace}'" else diff --git a/scripts/commit-msg.py b/scripts/commit-msg.py index 660c328b5..7de6dc3df 100755 --- a/scripts/commit-msg.py +++ b/scripts/commit-msg.py @@ -18,17 +18,17 @@ MSG_FILE = sys.argv[1] # commit-msg hook provides file name as first arg -def appendToCommitMessage(jiraId): - print("Appended {0} to commit message.".format(jiraId)) +def append_to_commit_message(jira_id): + print("Appended {0} to commit message.".format(jira_id)) with open(MSG_FILE, "a") as f: - f.write("{0}".format(jiraId)) + f.write("{0}".format(jira_id)) -def prependToCommitMessage(jiraId): - print("Prepended {} to commit message.".format(jiraId)) +def prepend_to_commit_message(jira_id): + print("Prepended {} to commit message.".format(jira_id)) with open(MSG_FILE, "r") as f: text = f.read() - text = "{} {}".format(jiraId, text) + text = "{} {}".format(jira_id, text) with open(MSG_FILE, "w") as f: f.write(text) @@ -65,23 +65,23 @@ def prependToCommitMessage(jiraId): ) ) if response.strip().lower().startswith("y"): - prependToCommitMessage(branchMatch.group(1)) + prepend_to_commit_message(branchMatch.group(1)) sys.exit(0) oldBranchMatch = re.search(OLD_FEATURE_REGEX, str(branch)) if oldBranchMatch: - jiraId = "SPII-{}".format(oldBranchMatch.group(1)) + jira_id = "SPII-{}".format(oldBranchMatch.group(1)) response = input( - "Use {0}{1}{2} from branch name? (y/n) ".format(OK_C, jiraId, END_C) + "Use {0}{1}{2} from branch name? (y/n) ".format(OK_C, jira_id, END_C) ) if response.strip().lower().startswith("y"): - prependToCommitMessage(jiraId) + prepend_to_commit_message(jira_id) sys.exit(0) # All else fails ask what they want to use -jiraId = input("Please enter the JIRA ID (for example SPII-1234 or NEMS-123): ") -if not re.search(JIRA_REGEX, jiraId): +jira_id = input("Please enter the JIRA ID (for example SPII-1234 or NEMS-123): ") +if not re.search(JIRA_REGEX, jira_id): print(FAIL_C + "Did not provide valid JIRA ID. Commit Aborted." + END_C) sys.exit(1) else: - prependToCommitMessage(jiraId) + prepend_to_commit_message(jira_id) sys.exit(0) diff --git a/scripts/get-account-name-for-env.sh b/scripts/get-account-name-for-env.sh index 2c377b6ae..28ff1a3f8 100755 --- a/scripts/get-account-name-for-env.sh +++ b/scripts/get-account-name-for-env.sh @@ -2,7 +2,7 @@ # Get the account name for the provided NRLF environment set -o errexit -o nounset -o pipefail -if [ $# -ne 1 ]; then +if [[ $# -ne 1 ]]; then echo "Usage: get-account-name-for-env.sh " exit 1 fi diff --git a/scripts/get-current-info.sh b/scripts/get-current-info.sh index 5140791dc..b3dc69140 100755 --- a/scripts/get-current-info.sh +++ b/scripts/get-current-info.sh @@ -6,7 +6,7 @@ BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" TAG_NAME="$(git describe --tags || echo "no-tags")" SHORT_COMMIT_HASH="$(git rev-parse --short=8 HEAD)" -if [ "${BRANCH_NAME}" == "HEAD" ]; then +if [[ "${BRANCH_NAME}" == "HEAD" ]]; then NRLF_VERSION="${TAG_NAME}@${SHORT_COMMIT_HASH}" else NRLF_VERSION="${BRANCH_NAME}@${SHORT_COMMIT_HASH}" diff --git a/scripts/pull-lambda-code-for-stack.sh b/scripts/pull-lambda-code-for-stack.sh index 4e497160e..91a0fca87 100755 --- a/scripts/pull-lambda-code-for-stack.sh +++ b/scripts/pull-lambda-code-for-stack.sh @@ -4,7 +4,7 @@ set -o errexit -o nounset -o pipefail : "${DIST_DIR:="./dist"}" -if [ $# -ne 1 ] +if [[ $# -ne 1 ]] then echo "Error: stack-name argument is missing" 1>&2 echo "Usage: $0 " 1>&2 @@ -22,7 +22,9 @@ function pull_lambda_code(){ echo -n "- Downloading code for lambda ${lambda_name}.... " code_url="$(aws lambda get-function --function-name "${lambda_name}" | jq -r .Code.Location)" curl "${code_url}" 2>/dev/null > "${DIST_DIR}/${api_name}-${endpoint_name}.zip" + echo "✅" + return 0 } function pull_layer_code(){ @@ -35,7 +37,9 @@ function pull_layer_code(){ echo -n "- Downloading code for layer ${layer_name} version ${layer_version}.... " code_url="$(aws lambda get-layer-version --layer-name "${layer_name}" --version-number "${layer_version}" | jq -r .Content.Location)" curl "${code_url}" 2>/dev/null > "${DIST_DIR}/${layer_pkg_name}" + echo "✅" + return 0 } mkdir -p "${DIST_DIR}" @@ -44,7 +48,7 @@ echo echo "Pulling code for consumer API lambdas...." for endpoint_path in api/consumer/* do - if [ ! -d "${endpoint_path}" ] + if [[ ! -d "${endpoint_path}" ]] then continue fi @@ -57,7 +61,7 @@ echo echo "Pulling code for producer API lambdas...." for endpoint_path in api/producer/* do - if [ ! -d "${endpoint_path}" ] + if [[ ! -d "${endpoint_path}" ]] then continue fi diff --git a/scripts/seed_nft_tables.py b/scripts/seed_nft_tables.py index 5c5d118bd..5ad023ebb 100644 --- a/scripts/seed_nft_tables.py +++ b/scripts/seed_nft_tables.py @@ -241,14 +241,20 @@ def _set_up_cyclical_iterator(dists: dict[str, int]) -> Iterator[str]: value_list: list[str] = [] for entry in dists: value_list.extend([entry] * (dists[entry] // d)) - shuffle(value_list) + shuffle(value_list) # NOSONAR (S2245) - psuedorandom shuffle is ok return cycle(value_list) def _get_pointer_count_poisson_distributions( num_of_patients: int, pointers_per_px: float ) -> Iterator[int]: - p_count_distr = np.random.poisson(lam=pointers_per_px - 1, size=num_of_patients) + 1 + rng_seed = int(datetime.now().timestamp()) + p_count_distr = ( + np.random.default_rng(rng_seed).poisson( + lam=pointers_per_px - 1, size=num_of_patients + ) + + 1 + ) p_count_distr = np.clip(p_count_distr, a_min=1, a_max=4) return cycle(p_count_distr) diff --git a/scripts/set_smoketest_permissions.py b/scripts/set_smoketest_permissions.py index af2f7df1b..3c8f196dd 100755 --- a/scripts/set_smoketest_permissions.py +++ b/scripts/set_smoketest_permissions.py @@ -1,18 +1,19 @@ #!/usr/bin/env python import json -from os import path -from pathlib import Path +import os import fire from aws_session_assume import get_boto_session +AWS_REGION = os.getenv("AWS_REGION", "eu-west-2") + def main(secret_env_name: str = "dev", bucket_env_name: str = "dev", env: str = "dev"): boto_session = get_boto_session(env) print("Getting smoke test parameters from AWS....") # noqa smoke_test_params_name = f"nhsd-nrlf--{secret_env_name}--smoke-test-parameters" - secretsmanager = boto_session.client("secretsmanager", region_name="eu-west-2") + secretsmanager = boto_session.client("secretsmanager", region_name=AWS_REGION) smoke_test_params_value = secretsmanager.get_secret_value( SecretId=smoke_test_params_name ) @@ -29,6 +30,9 @@ def main(secret_env_name: str = "dev", bucket_env_name: str = "dev", env: str = Bucket=bucket, Key=f"{nrlf_app_id}/{ods_code}.json", Body=open(f"./tests/smoke/permissions/{ods_code}.json", "rb"), + ExpectedBucketOwner=boto_session.client("sts") + .get_caller_identity() + .get("Account"), ) diff --git a/scripts/truststore.sh b/scripts/truststore.sh index 97fa82b8b..e65bd2eac 100755 --- a/scripts/truststore.sh +++ b/scripts/truststore.sh @@ -34,6 +34,7 @@ function _truststore_help() { echo " restore-archived-ca - restore an archived certificate authority" echo " restore-archived-cert - restore an archived client certificate" echo + return 0 } # read an input file and substitute all the ${} entries @@ -400,6 +401,8 @@ function _truststore() { "restore-archived-cert") _restore_archived_cert $args ;; *) _truststore_help $args ;; esac + + return 0 } _truststore $@ diff --git a/sonar-project.properties b/sonar-project.properties index 746c4e678..0eb2d19af 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,10 +5,10 @@ sonar.projectName=NRLF sonar.python.version=3.12 sonar.sources=. -sonar.exclusions=scripts/** sonar.tests=. -sonar.tests.inclusions=**/tests/** -sonar.coverage.exclusions=scripts/**, tests/**, **/tests/** -sonar.cpd.exclusions=tests/**, **/tests/** +sonar.test.inclusions=**/tests/** +sonar.coverage.inclusions=**/*.py +sonar.coverage.exclusions=scripts/**,tests/**, **/tests/**, **/*_schemas.py +sonar.cpd.exclusions=tests/**, **/tests/**, **/*_schemas.py sonar.python.coverage.reportPaths=dist/test-coverage.xml diff --git a/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql b/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql index 53c968905..01add7f2f 100644 --- a/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql +++ b/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql @@ -33,49 +33,10 @@ WITH , event_function_request_id , event_correlation_id , event_xray_trace_id - --, event_pointer_types , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_deletedocumentreference ) -/*, pr AS ( - SELECT - time - , event_timestamp - , date - , host - , event_log_reference - , event_level - , event_location - , event_message - , event_service - , event_function_request_id - , event_correlation_id - , event_xray_trace_id - , event_pointer_types - , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods - FROM - producer_readdocumentreference -)*/ -/*, ps AS ( - SELECT - time - , event_timestamp - , date - , host - , event_log_reference - , event_level - , event_location - , event_message - , event_service - , event_function_request_id - , event_correlation_id - , event_xray_trace_id - , event_pointer_types - , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods - FROM - producer_searchdocumentreference -)*/ , psp AS ( SELECT time @@ -90,30 +51,10 @@ WITH , event_function_request_id , event_correlation_id , event_xray_trace_id - --, event_pointer_types , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_searchpostdocumentreference ) -/*, pu AS ( - SELECT - time - , event_timestamp - , date - , host - , event_log_reference - , event_level - , event_location - , event_message - , event_service - , event_function_request_id - , event_correlation_id - , event_xray_trace_id - , event_pointer_types - , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods - FROM - producer_updatedocumentreference -)*/ , pus AS ( SELECT time @@ -128,7 +69,6 @@ WITH , event_function_request_id , event_correlation_id , event_xray_trace_id - --, event_pointer_types , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_upsertdocumentreference @@ -140,18 +80,9 @@ WITH UNION SELECT * FROM pd -/*UNION SELECT * - FROM - pr -UNION SELECT * - FROM - ps*/ UNION SELECT * FROM psp -/*UNION SELECT * - FROM - pu*/ UNION SELECT * FROM pus @@ -177,7 +108,6 @@ SELECT , event_function_request_id , b.event_correlation_id , b.event_xray_trace_id ---, event_pointer_types , oc.user_ods FROM (base b diff --git a/terraform/account-wide-infrastructure/modules/glue/LogSchemaGeneration/LogSchemaGeneration.ipynb b/terraform/account-wide-infrastructure/modules/glue/LogSchemaGeneration/LogSchemaGeneration.ipynb index 0c22ab055..ccf05e0c2 100644 --- a/terraform/account-wide-infrastructure/modules/glue/LogSchemaGeneration/LogSchemaGeneration.ipynb +++ b/terraform/account-wide-infrastructure/modules/glue/LogSchemaGeneration/LogSchemaGeneration.ipynb @@ -12,7 +12,8 @@ ")\n", "import json\n", "from datetime import datetime, timedelta\n", - "from typing import List, Dict, Type, Set, Any" + "import os\n", + "from typing import List, Dict, Any" ] }, { @@ -22,7 +23,8 @@ "outputs": [], "source": [ "# Initialize CloudWatch client\n", - "session = boto3.Session(region_name=\"eu-west-2\", profile_name=\"nhsd-nrlf-dev\")\n", + "AWS_REGION = os.getenv(\"AWS_REGION\", \"eu-west-2\")\n", + "session = boto3.Session(profile_name=\"nhsd-nrlf-dev\", region_name=AWS_REGION)\n", "client = session.client(\"logs\")" ] }, diff --git a/terraform/bastion/scripts/start-bastion-connection.sh b/terraform/bastion/scripts/start-bastion-connection.sh index 932427ccd..27fda699d 100755 --- a/terraform/bastion/scripts/start-bastion-connection.sh +++ b/terraform/bastion/scripts/start-bastion-connection.sh @@ -5,7 +5,7 @@ set -o errexit -o nounset -o pipefail : "${AWS_ACCOUNT_ID:=""}" : "${AWS_ROLE_NAME:=""}" -if [ "$#" -ne 1 ]; then +if [[ "$#" -ne 1 ]]; then echo "Usage: $0 " exit 1 fi