From 63b9f0a679d4e1ad121d0874a420f87dc2d8d2ac Mon Sep 17 00:00:00 2001
From: Harry Stead <Harry.Stead@mft.nhs.uk>
Date: Mon, 27 Mar 2023 17:02:48 +0100
Subject: [PATCH 1/2] Log infrastructure added

---
 .gitignore                                    |   3 +-
 deploy/forwardLogs/cloudwatch.tf              |   4 +
 .../lambda-forward-logs-s3-cloudwatch.json    |  28 +++
 .../forward-logs-cloudwatch-lambda/index.js   | 228 ++++++++++++++++++
 deploy/forwardLogs/iam-role.tf                |  57 +++++
 deploy/forwardLogs/lambda.tf                  |  45 ++++
 deploy/forwardLogs/variables.tf               |  23 ++
 7 files changed, 387 insertions(+), 1 deletion(-)
 create mode 100644 deploy/forwardLogs/cloudwatch.tf
 create mode 100644 deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json
 create mode 100644 deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js
 create mode 100644 deploy/forwardLogs/iam-role.tf
 create mode 100644 deploy/forwardLogs/lambda.tf
 create mode 100644 deploy/forwardLogs/variables.tf

diff --git a/.gitignore b/.gitignore
index 01e20bf9..b9174981 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ cypress/videos
 cypress/screenshots
 
 # Terraform
-deploy/*terraform*
+deploy/*/*terraform*
+archive.zip
 
 
diff --git a/deploy/forwardLogs/cloudwatch.tf b/deploy/forwardLogs/cloudwatch.tf
new file mode 100644
index 00000000..2ee0549a
--- /dev/null
+++ b/deploy/forwardLogs/cloudwatch.tf
@@ -0,0 +1,4 @@
+resource "aws_cloudwatch_log_group" "alb_access_log_group" {
+  name = "${var.environment}-cloudfront-react-log"
+  retention_in_days = 400
+}
diff --git a/deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json b/deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json
new file mode 100644
index 00000000..f6f5b943
--- /dev/null
+++ b/deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json
@@ -0,0 +1,28 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:DescribeLogGroups",
+        "logs:DescribeLogStreams",
+        "logs:PutLogEvents",
+        "logs:GetLogEvents",
+        "logs:FilterLogEvents",
+        "ec2:CreateNetworkInterface",
+        "ec2:DescribeNetworkInterfaces",
+        "ec2:DeleteNetworkInterface"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Action": [
+        "s3:GetObject"
+      ],
+      "Effect": "Allow",
+      "Resource": ["*"]
+    }
+  ]
+}
diff --git a/deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js b/deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js
new file mode 100644
index 00000000..fda4071d
--- /dev/null
+++ b/deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js
@@ -0,0 +1,228 @@
+const aws = require('aws-sdk');
+const zlib = require('zlib');
+const s3 = new aws.S3({ apiVersion: '2006-03-01' });
+const cloudWatchLogs = new aws.CloudWatchLogs({
+  apiVersion: '2014-03-28',
+});
+
+const readline = require('readline');
+const stream = require('stream');
+
+let logGroupName = process.env.logGroupName;
+let logStreamName;
+
+exports.handler = (event, context, callback) => {
+
+  logStreamName = context.logStreamName;
+  console.log('S3 object is:', event.Records[0].s3);
+  const bucket = event.Records[0].s3.bucket.name;
+  console.log('Name of S3 bucket is:', bucket);
+  const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
+
+  const params = {
+    Bucket: bucket,
+    Key: key,
+  };
+
+  s3.getObject(params, (err, data) => {
+    if (err) {
+      console.log(err);
+      const message = `Error getting object ${key} from bucket ${bucket}. Make sure they exist and your bucket is in the same region as this function.`;
+      console.log(message);
+      callback(message);
+    } else {
+      zlib.gunzip(data.Body, function (error, buffer) {
+        if (error) {
+          console.log('Error uncompressing data', error);
+          return;
+        }
+
+        const logData = buffer.toString('ascii');
+        manageLogGroups(logData);
+      });
+      callback(null, data.ContentType);
+    }
+  });
+
+  function manageLogGroups(logData) {
+    const describeLogGroupParams = {
+      logGroupNamePrefix: logGroupName,
+    };
+
+    cloudWatchLogs.describeLogGroups(describeLogGroupParams, function (err, data) {
+      if (err) {
+        console.log('Error while describing log group:', err);
+        createLogGroup(logData);
+      } else {
+        if (!data.logGroups[0]) {
+          console.log('Need to  create log group:', data);
+          createLogGroup(logData);
+        } else {
+          console.log('Success while describing log group:', data);
+          manageLogStreams(logData);
+        }
+      }
+    });
+  }
+
+  function createLogGroup(logData) {
+    const logGroupParams = {
+      logGroupName: logGroupName,
+    };
+    cloudWatchLogs.createLogGroup(logGroupParams, function (err, data) {
+      if (err) {
+        console.log('error while creating log group: ', err, err.stack);
+        return;
+      } else {
+        console.log('Success in creating log group: ', logGroupName);
+        manageLogStreams(logData);
+      }
+    });
+  }
+
+  function manageLogStreams(logData) {
+    const describeLogStreamsParams = {
+      logGroupName: logGroupName,
+      logStreamNamePrefix: logStreamName,
+    };
+
+    cloudWatchLogs.describeLogStreams(describeLogStreamsParams, function (err, data) {
+      if (err) {
+        console.log('Error during describe log streams:', err);
+        createLogStream(logData);
+      } else {
+        if (!data.logStreams[0]) {
+          console.log('Need to  create log stream:', data);
+          createLogStream(logData);
+        } else {
+          console.log('Log Stream already defined:', logStreamName);
+          putLogEvents(data.logStreams[0].uploadSequenceToken, logData);
+        }
+      }
+    });
+  }
+
+  function createLogStream(logData) {
+    const logStreamParams = {
+      logGroupName: logGroupName,
+      logStreamName: logStreamName,
+    };
+
+    cloudWatchLogs.createLogStream(logStreamParams, function (err, data) {
+      if (err) {
+        console.log('error while creating log stream: ', err, err.stack);
+        return;
+      } else {
+        console.log('Success in creating log stream: ', logStreamName);
+        putLogEvents(null, logData);
+      }
+    });
+  }
+
+  function putLogEvents(sequenceToken, logData) {
+    //From http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
+    const MAX_BATCH_SIZE = 1048576; // maximum size in bytes of Log Events (with overhead) per invocation of PutLogEvents
+    const MAX_BATCH_COUNT = 10000; // maximum number of Log Events per invocation of PutLogEvents
+    const LOG_EVENT_OVERHEAD = 26; // bytes of overhead per Log Event
+
+    const batches = [];
+
+    let batch = [];
+
+    let batch_size = 0;
+
+    const bufferStream = new stream.PassThrough();
+    bufferStream.end(logData);
+
+    const rl = readline.createInterface({
+      input: bufferStream,
+    });
+
+    let line_count = 0;
+
+    rl.on('line', (line) => {
+      ++line_count;
+      if (line[0] != "#") {
+        let timeValue = Date.parse(line.split('\t')[0]);
+
+        const event_size = line.length + LOG_EVENT_OVERHEAD;
+
+        batch_size += event_size;
+
+        if (batch_size >= MAX_BATCH_SIZE || batch.length >= MAX_BATCH_COUNT) {
+          batches.push(batch);
+          batch = [];
+          batch_size = event_size;
+        }
+
+        batch.push({
+          message: line,
+          timestamp: timeValue,
+        });
+      }
+    });
+
+    rl.on('close', () => {
+      if (batch.length > 0) {
+        batches.push(batch);
+    }
+    sendBatches(sequenceToken, batches);
+    });
+  }
+
+  function sendBatches(sequenceToken, batches) {
+    let count = 0;
+    let batch_count = 0;
+
+    function sendNextBatch(err, nextSequenceToken) {
+      if (err) {
+        console.log('Error sending batch: ', err, err.stack);
+        return;
+      } else {
+        const nextBatch = batches.shift();
+        if (nextBatch) {
+          ++batch_count;
+          count += nextBatch.length;
+          sendBatch(nextSequenceToken, nextBatch, sendNextBatch);
+        } else {
+          const msg = `Successfully put ${count} events in ${batch_count} batches`;
+          console.log(msg);
+          callback(null, msg);
+        }
+      }
+    }
+
+    sendNextBatch(null, sequenceToken);
+  }
+
+  function sendBatch(sequenceToken, batch, doNext) {
+    const putLogEventParams = {
+      logEvents: batch,
+      logGroupName: logGroupName,
+      logStreamName: logStreamName,
+    };
+    if (sequenceToken) {
+      putLogEventParams['sequenceToken'] = sequenceToken;
+    }
+
+    putLogEventParams.logEvents.sort(function (a, b) {
+      if (a.timestamp > b.timestamp) {
+        return 1;
+      }
+      if (a.timestamp < b.timestamp) {
+        return -1;
+      }
+      return 0;
+    });
+
+    cloudWatchLogs.putLogEvents(putLogEventParams, function (err, data) {
+      if (err) {
+        console.log('Error during put log events: ', err, err.stack);
+        doNext(err, null);
+      } else {
+        console.log(`Success in putting ${putLogEventParams.logEvents.length} events`);
+        doNext(null, data.nextSequenceToken);
+      }
+    });
+  }
+};
diff --git a/deploy/forwardLogs/iam-role.tf b/deploy/forwardLogs/iam-role.tf
new file mode 100644
index 00000000..5d00191d
--- /dev/null
+++ b/deploy/forwardLogs/iam-role.tf
@@ -0,0 +1,57 @@
+resource "aws_iam_role" "lambda_forward_logs_s3_cloudwatch_role" {
+  name                 = "${var.environment}-lambda-forward-logs-s3-cloudwatch-role"
+
+  assume_role_policy = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "Service": "lambda.amazonaws.com"
+            },
+            "Action": "sts:AssumeRole"
+        }
+    ]
+}
+EOF
+  tags = merge({
+    Name = "${var.environment}-iam-role"
+  },
+  )
+}
+
+resource "aws_iam_role_policy" "lambda_forward_logs_s3_cloudwatch_role_policy" {
+  name   = "${var.environment}-lambda-forward-logs-s3-cloudwatch-policy"
+  role   = aws_iam_role.lambda_forward_logs_s3_cloudwatch_role.id
+  policy = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Action": [
+          "logs:CreateLogGroup",
+          "logs:CreateLogStream",
+          "logs:DescribeLogGroups",
+          "logs:DescribeLogStreams",
+          "logs:PutLogEvents",
+          "logs:GetLogEvents",
+          "logs:FilterLogEvents",
+          "ec2:CreateNetworkInterface",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DeleteNetworkInterface"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Action": [
+          "s3:GetObject"
+        ],
+        "Effect": "Allow",
+        "Resource": ["*"]
+      }
+    ]
+  }
+  EOF
+}
diff --git a/deploy/forwardLogs/lambda.tf b/deploy/forwardLogs/lambda.tf
new file mode 100644
index 00000000..fd1ecb81
--- /dev/null
+++ b/deploy/forwardLogs/lambda.tf
@@ -0,0 +1,45 @@
+data "archive_file" "archive_pipe_logs_s3_cloudwatch_lambda" {
+    type        = "zip"
+    source_file = "${path.module}/forward-logs-cloudwatch-lambda/index.js"
+    output_path = "${path.module}/forward-logs-cloudwatch-lambda/archive.zip"
+}
+
+resource "aws_lambda_function" "forward_logs_s3_cloudwatch" {
+    filename         = "${path.module}/forward-logs-cloudwatch-lambda/archive.zip"
+    function_name    = "${var.environment}-forward-cloudfront-logs"
+    role             = aws_iam_role.lambda_forward_logs_s3_cloudwatch_role.arn
+    handler          = "index.handler"
+    runtime          = "nodejs14.x"
+    timeout       = 30
+
+    source_code_hash = data.archive_file.archive_pipe_logs_s3_cloudwatch_lambda.output_base64sha256
+
+    tracing_config {
+        mode = "Active"
+    }
+
+    environment {
+        variables = {
+            environment = var.environment
+            logGroupName = aws_cloudwatch_log_group.alb_access_log_group.name
+        }
+    }
+}
+
+resource "aws_lambda_permission" "allow_bucket_forward_logs" {
+    statement_id  = "AllowExecutionFromS3Bucket"
+    action        = "lambda:InvokeFunction"
+    function_name = aws_lambda_function.forward_logs_s3_cloudwatch.arn
+    principal     = "s3.amazonaws.com"
+    source_arn    = var.s3_source_arn_event_notification
+    source_account = var.account_id
+}
+
+resource "aws_s3_bucket_notification" "aws-lambda-trigger-alb-cloudfront" {
+    depends_on = [aws_lambda_permission.allow_bucket_forward_logs]
+    bucket = var.s3_bucket_name
+    lambda_function {
+        lambda_function_arn = aws_lambda_function.forward_logs_s3_cloudwatch.arn
+        events              = ["s3:ObjectCreated:*"]
+    }
+}
diff --git a/deploy/forwardLogs/variables.tf b/deploy/forwardLogs/variables.tf
new file mode 100644
index 00000000..cf88ad8f
--- /dev/null
+++ b/deploy/forwardLogs/variables.tf
@@ -0,0 +1,23 @@
+variable "environment" {
+  type        = string
+  description = "Environment name or equivalent for CI CD and resource naming purpose."
+  default     = "dev"
+}
+
+variable "s3_source_arn_event_notification" {
+  type        = string
+  description = "The S3 ARN that will trigger the lambda through even notification"
+  default     = ""
+}
+
+variable "s3_bucket_name" {
+  type        = string
+  description = "The S3 bucket name"
+  default     = ""
+}
+
+variable "account_id" {
+  type        = string
+  description = "Account Id"
+  default     = ""
+}

From 1faef5bfa76d24a0a361ced76f361542d5e2da00 Mon Sep 17 00:00:00 2001
From: Harry Stead <Harry.Stead@mft.nhs.uk>
Date: Mon, 27 Mar 2023 17:07:10 +0100
Subject: [PATCH 2/2] formatting

---
 deploy/forwardLogs/cloudwatch.tf              |  2 +-
 .../lambda-forward-logs-s3-cloudwatch.json    |  4 +-
 .../forward-logs-cloudwatch-lambda/index.js   |  7 +--
 deploy/forwardLogs/iam-role.tf                |  4 +-
 deploy/forwardLogs/lambda.tf                  | 60 +++++++++----------
 package.json                                  |  4 +-
 6 files changed, 39 insertions(+), 42 deletions(-)

diff --git a/deploy/forwardLogs/cloudwatch.tf b/deploy/forwardLogs/cloudwatch.tf
index 2ee0549a..a9481da5 100644
--- a/deploy/forwardLogs/cloudwatch.tf
+++ b/deploy/forwardLogs/cloudwatch.tf
@@ -1,4 +1,4 @@
 resource "aws_cloudwatch_log_group" "alb_access_log_group" {
-  name = "${var.environment}-cloudfront-react-log"
+  name              = "${var.environment}-cloudfront-react-log"
   retention_in_days = 400
 }
diff --git a/deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json b/deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json
index f6f5b943..22d6a304 100644
--- a/deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json
+++ b/deploy/forwardLogs/config/lambda-forward-logs-s3-cloudwatch.json
@@ -18,9 +18,7 @@
       "Resource": "*"
     },
     {
-      "Action": [
-        "s3:GetObject"
-      ],
+      "Action": ["s3:GetObject"],
       "Effect": "Allow",
       "Resource": ["*"]
     }
diff --git a/deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js b/deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js
index fda4071d..d18fc033 100644
--- a/deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js
+++ b/deploy/forwardLogs/forward-logs-cloudwatch-lambda/index.js
@@ -12,7 +12,6 @@ let logGroupName = process.env.logGroupName;
 let logStreamName;
 
 exports.handler = (event, context, callback) => {
-
   logStreamName = context.logStreamName;
   console.log('S3 object is:', event.Records[0].s3);
   const bucket = event.Records[0].s3.bucket.name;
@@ -142,7 +141,7 @@ exports.handler = (event, context, callback) => {
 
     rl.on('line', (line) => {
       ++line_count;
-      if (line[0] != "#") {
+      if (line[0] != '#') {
         let timeValue = Date.parse(line.split('\t')[0]);
 
         const event_size = line.length + LOG_EVENT_OVERHEAD;
@@ -165,8 +164,8 @@ exports.handler = (event, context, callback) => {
     rl.on('close', () => {
       if (batch.length > 0) {
         batches.push(batch);
-    }
-    sendBatches(sequenceToken, batches);
+      }
+      sendBatches(sequenceToken, batches);
     });
   }
 
diff --git a/deploy/forwardLogs/iam-role.tf b/deploy/forwardLogs/iam-role.tf
index 5d00191d..b856fcaf 100644
--- a/deploy/forwardLogs/iam-role.tf
+++ b/deploy/forwardLogs/iam-role.tf
@@ -1,5 +1,5 @@
 resource "aws_iam_role" "lambda_forward_logs_s3_cloudwatch_role" {
-  name                 = "${var.environment}-lambda-forward-logs-s3-cloudwatch-role"
+  name = "${var.environment}-lambda-forward-logs-s3-cloudwatch-role"
 
   assume_role_policy = <<EOF
 {
@@ -17,7 +17,7 @@ resource "aws_iam_role" "lambda_forward_logs_s3_cloudwatch_role" {
 EOF
   tags = merge({
     Name = "${var.environment}-iam-role"
-  },
+    },
   )
 }
 
diff --git a/deploy/forwardLogs/lambda.tf b/deploy/forwardLogs/lambda.tf
index fd1ecb81..e751f001 100644
--- a/deploy/forwardLogs/lambda.tf
+++ b/deploy/forwardLogs/lambda.tf
@@ -1,45 +1,45 @@
 data "archive_file" "archive_pipe_logs_s3_cloudwatch_lambda" {
-    type        = "zip"
-    source_file = "${path.module}/forward-logs-cloudwatch-lambda/index.js"
-    output_path = "${path.module}/forward-logs-cloudwatch-lambda/archive.zip"
+  type        = "zip"
+  source_file = "${path.module}/forward-logs-cloudwatch-lambda/index.js"
+  output_path = "${path.module}/forward-logs-cloudwatch-lambda/archive.zip"
 }
 
 resource "aws_lambda_function" "forward_logs_s3_cloudwatch" {
-    filename         = "${path.module}/forward-logs-cloudwatch-lambda/archive.zip"
-    function_name    = "${var.environment}-forward-cloudfront-logs"
-    role             = aws_iam_role.lambda_forward_logs_s3_cloudwatch_role.arn
-    handler          = "index.handler"
-    runtime          = "nodejs14.x"
-    timeout       = 30
+  filename      = "${path.module}/forward-logs-cloudwatch-lambda/archive.zip"
+  function_name = "${var.environment}-forward-cloudfront-logs"
+  role          = aws_iam_role.lambda_forward_logs_s3_cloudwatch_role.arn
+  handler       = "index.handler"
+  runtime       = "nodejs14.x"
+  timeout       = 30
 
-    source_code_hash = data.archive_file.archive_pipe_logs_s3_cloudwatch_lambda.output_base64sha256
+  source_code_hash = data.archive_file.archive_pipe_logs_s3_cloudwatch_lambda.output_base64sha256
 
-    tracing_config {
-        mode = "Active"
-    }
+  tracing_config {
+    mode = "Active"
+  }
 
-    environment {
-        variables = {
-            environment = var.environment
-            logGroupName = aws_cloudwatch_log_group.alb_access_log_group.name
-        }
+  environment {
+    variables = {
+      environment  = var.environment
+      logGroupName = aws_cloudwatch_log_group.alb_access_log_group.name
     }
+  }
 }
 
 resource "aws_lambda_permission" "allow_bucket_forward_logs" {
-    statement_id  = "AllowExecutionFromS3Bucket"
-    action        = "lambda:InvokeFunction"
-    function_name = aws_lambda_function.forward_logs_s3_cloudwatch.arn
-    principal     = "s3.amazonaws.com"
-    source_arn    = var.s3_source_arn_event_notification
-    source_account = var.account_id
+  statement_id   = "AllowExecutionFromS3Bucket"
+  action         = "lambda:InvokeFunction"
+  function_name  = aws_lambda_function.forward_logs_s3_cloudwatch.arn
+  principal      = "s3.amazonaws.com"
+  source_arn     = var.s3_source_arn_event_notification
+  source_account = var.account_id
 }
 
 resource "aws_s3_bucket_notification" "aws-lambda-trigger-alb-cloudfront" {
-    depends_on = [aws_lambda_permission.allow_bucket_forward_logs]
-    bucket = var.s3_bucket_name
-    lambda_function {
-        lambda_function_arn = aws_lambda_function.forward_logs_s3_cloudwatch.arn
-        events              = ["s3:ObjectCreated:*"]
-    }
+  depends_on = [aws_lambda_permission.allow_bucket_forward_logs]
+  bucket     = var.s3_bucket_name
+  lambda_function {
+    lambda_function_arn = aws_lambda_function.forward_logs_s3_cloudwatch.arn
+    events              = ["s3:ObjectCreated:*"]
+  }
 }
diff --git a/package.json b/package.json
index a0031864..52b7c213 100644
--- a/package.json
+++ b/package.json
@@ -84,7 +84,7 @@
     "npm": "^9 || ^8"
   },
   "lint-staged": {
-    "src/**/*.{js,jsx,ts,tsx}": "npm run formatPretty",
-    "src/**/*.{js,jsx,ts,tsx,css,scss,md}": "npm run formatLint"
+    "src/**/*.{js,jsx,ts,tsx,tf}": "npm run formatPretty",
+    "src/**/*.{js,jsx,ts,tsx,css,scss,md,tf}": "npm run formatLint"
   }
 }