CCM-16644: Record Events

.github/workflows/stage-2-test.yaml

@@ -40,39 +40,60 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      - name: "Use Node.js"
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version-file: '.tool-versions'
+      - name: "Install dependencies"
+        run: npm ci
       - name: "Run unit test suite"
         run: |
           make test-unit
       - name: "Save the result of fast test suite"
-        run: |
-          echo "Nothing to save"
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: unit-tests
+          path: "**/.reports/unit"
+          include-hidden-files: true
+        if: always()
+      - name: "Save the result of code coverage"
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: code-coverage-report
+          path: ".reports/lcov.info"
+        if: always()
   test-lint:
     name: "Linting"
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - name: "Checkout code"
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      - name: "Use Node.js"
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version-file: '.tool-versions'
+      - name: "Install dependencies"
+        run: npm ci
       - name: "Run linting"
         run: |
           make test-lint
-      - name: "Save the linting result"
-        run: |
-          echo "Nothing to save"
-  test-coverage:
-    name: "Test coverage"
-    needs: [test-unit]
+  test-typecheck:
+    name: "Typecheck"
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - name: "Checkout code"
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
-      - name: "Run test coverage check"
-        run: |
-          make test-coverage
-      - name: "Save the coverage check result"
+      - name: "Use Node.js"
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version-file: '.tool-versions'
+      - name: "Install dependencies"
+        run: npm ci
+      - name: "Run typecheck"
         run: |
-          echo "Nothing to save"
+          make test-typecheck
   perform-static-analysis:
     name: "Perform static analysis"
     needs: [test-unit]
@@ -86,8 +107,12 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
           fetch-depth: 0 # Full history is needed to improving relevancy of reporting
+      - name: "Download coverage report for SONAR"
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: code-coverage-report
       - name: "Perform static analysis"
-        uses: NHSDigital/nhs-notify-shared-modules/.github/actions/perform-static-analysis@3.1.4
+        uses: NHSDigital/nhs-notify-shared-modules/.github/actions/perform-static-analysis@740afa9b7c15fc8c44681f958ea8608062e77a25 # v3.1.4
         with:
           sonar_organisation_key: "${{ vars.SONAR_ORGANISATION_KEY }}"
           sonar_project_key: "${{ vars.SONAR_PROJECT_KEY }}"

.gitleaksignore

@@ -4,3 +4,9 @@
 39565cc5ab1245e4e6a6368c19fd0aa9a187733a:infrastructure/terraform/components/reporting/variables.tf:ipv4:109
 ca243cb73d3804a14f3eeefa8073c96802420c52:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
 ca243cb73d3804a14f3eeefa8073c96802420c52:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43
+e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
+e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43
+eb98e30495f24e1c27b438da069bad7d938d1c02:infrastructure/terraform/components/reporting/README.md:ipv4:16
+eb98e30495f24e1c27b438da069bad7d938d1c02:infrastructure/terraform/components/reporting/variables.tf:ipv4:109
+b476ac7da6815b396a0b7bf338f211aec32553bf:infrastructure/terraform/components/powerbi-gateway/README.md:ipv4:14
+b476ac7da6815b396a0b7bf338f211aec32553bf:infrastructure/terraform/components/powerbi-gateway/variables.tf:ipv4:77

.tool-versions

@@ -3,8 +3,9 @@ act 0.2.64
 vale 3.6.0
 gitleaks 8.18.4
 jq 1.6
-nodejs 18.18.2
+nodejs 22.22.2
 pre-commit 3.6.0
+ruby 3.2.0
 terraform 1.10.1
 terraform-docs 0.19.0
 # trivy 0.69.3

Makefile

@@ -8,10 +8,11 @@ include scripts/init.mk
 # Example CI/CD targets are: dependencies, build, publish, deploy, clean, etc.
 
 dependencies: # Install dependencies needed to build and test the project @Pipeline
-	# TODO: Implement installation of your project dependencies
+	npm ci
 
 build: # Build the project artefact @Pipeline
 	(cd docs && make build)
+	npm run build:archive
 
 publish: # Publish the project artefact @Pipeline
 	# TODO: Implement the artefact publishing step
@@ -21,10 +22,12 @@ deploy: # Deploy the project artefact to the target environment @Pipeline
 
 clean:: # Clean-up project resources (main) @Operations
 	rm -f .version
-	# TODO: Implement project resources clean-up step
+	rm -rf .reports
+	npm run clean
 
 config:: _install-dependencies version # Configure development environment (main) @Configuration
 	(cd docs && make install)
+	mkdir -p .reports
 
 version:
 	rm -f .version

README.md

@@ -11,14 +11,17 @@ The reporting domain provides an isolated environment for staging data used for
 
 This allows appropriate views of data to be safely exposed without sharing the full contents of the underlying transactional database.
 
-This domain does not contain any application code. The reporting domain is executed exclusively through AWS services. It incorporates the following technologies:
+The reporting domain incorporates the following technologies:
 
 - [HashiCorp Terraform](https://developer.hashicorp.com/terraform)
 - [AWS Step Functions](https://docs.aws.amazon.com/step-functions/latest/dg/welcome.html)
 - [AWS Glue](https://docs.aws.amazon.com/glue/latest/dg/what-is-glue.html)
 - [AWS Athena](https://docs.aws.amazon.com/athena/latest/ug/what-is.html)
 - [AWS S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html)
 - [AWS Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html)
+- [AWS SNS](https://docs.aws.amazon.com/sns/latest/dg/welcome.html)
+- [AWS Data Firehose](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
+- [AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html)
 - [Apache Iceberg](https://iceberg.apache.org/)
 - [Microsoft Power BI](https://learn.microsoft.com/en-us/power-bi/)
 
@@ -57,6 +60,12 @@ git clone https://github.com/NHSDigital/nhs-notify-reporting.git
 cd nhs-notify-reporting
 ```
 
+Configure your development environment (requires [ASDF](https://asdf-vm.com/) to be installed):
+
+```shell
+make config
+```
+
 ## Prerequisites
 
 In order to facilitate cross-account export of data, the reporting domain requires IAM permissions for read-only access to the Glue Catalogue and underlying S3 storage in the NHS Notify core account.
@@ -134,7 +143,7 @@ New columns should also be added to the underlying [table definition](/infrastru
 
 ## Testing
 
-As there is no application code suited to unit testing, testing of step functions and queries is currently performed manually.
+All tests for the repository can be run using the `make test` command. You will need to have run `make config` first to configure your development environment.
 
 ## Design
 
@@ -191,6 +200,18 @@ This means that it is possible to run an aggregation on a previously executed pr
 
 This offers performance, scaling and cost benefits and also means that aggregation contents will be consistent with underlying projections.
 
+### Event Driven Reporting
+
+To allow reporting on data that originates in non-core domains, the following architecture has been implemented in addition to that described above:
+![Reporting Architecture Overview](/docs/diagrams/event_driven_reporting.png)
+
+Events are streamed into append-only storage using parquet, and then ingested periodically into reporting tables, effectively following the same approach as the current core ingestion.
+The event staging table is partitioned by the event's `type` field, then year, month and day, allowing for easy querying of the relevant event type(s).
+
+The event staging table's schema defines a column for each of the event envelope fields that are common across Notify domains, with the `data` column being defined as a string.
+The transformation lambda transforms each event's `data` property into a JSON string, allowing the solution to handle events of any shape. Event-specific ingestion queries can then be produced,
+with knowledge of the relevant event structures to transform them into the correct format and upsert them into a reporting table.
+
 ### Handling PID
 
 Staging tables exposed to Power BI should not contain any PID.