From 28a4ed6bb186cdd9b0487567f5463c58f5d601a0 Mon Sep 17 00:00:00 2001 From: vapor-forensics Date: Mon, 20 Jan 2025 16:18:22 +1000 Subject: [PATCH 1/2] dev --- apps/aws/migrations/0001_initial.py | 19 +- .../0002_alter_awsresource_aws_region.py | 18 -- apps/aws/migrations/0003_awslogsource.py | 30 --- apps/aws/urls.py | 3 +- apps/aws/utils.py | 212 ++++++++++-------- apps/aws/views.py | 73 ++++-- apps/case/urls.py | 1 - apps/case/views.py | 34 ++- apps/data/migrations/0001_initial.py | 28 ++- ...ove_normalizedlog_aws_resource_and_more.py | 28 --- .../migrations/0003_normalizedlog_log_id.py | 18 -- apps/data/models.py | 39 ++-- templates/account/login.html | 3 +- ...nt_details.html => account_resources.html} | 43 +++- templates/aws/normalized_logs.html | 118 ++++++++++ templates/case/case_detail.html | 100 +++++++-- templates/case/list_connected_accounts.html | 89 -------- templates/web/app/app_base.html | 11 +- templates/web/app_home.html | 58 ++--- templates/web/components/footer.html | 6 +- templates/web/components/top_nav.html | 2 +- 21 files changed, 530 insertions(+), 403 deletions(-) delete mode 100644 apps/aws/migrations/0002_alter_awsresource_aws_region.py delete mode 100644 apps/aws/migrations/0003_awslogsource.py delete mode 100644 apps/data/migrations/0002_remove_normalizedlog_aws_resource_and_more.py delete mode 100644 apps/data/migrations/0003_normalizedlog_log_id.py rename templates/aws/{account_details.html => account_resources.html} (64%) create mode 100644 templates/aws/normalized_logs.html delete mode 100644 templates/case/list_connected_accounts.html diff --git a/apps/aws/migrations/0001_initial.py b/apps/aws/migrations/0001_initial.py index 7be47ef..99ab1ad 100644 --- a/apps/aws/migrations/0001_initial.py +++ b/apps/aws/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.3 on 2024-12-25 11:56 +# Generated by Django 5.1.3 on 2025-01-20 04:31 import django.db.models.deletion from django.conf import settings @@ -29,6 +29,21 @@ class Migration(migrations.Migration): ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_accounts', to='case.case')), ], ), + migrations.CreateModel( + name='AWSLogSource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('service_name', models.CharField(max_length=100)), + ('log_name', models.CharField(max_length=255)), + ('log_details', models.JSONField(blank=True, null=True)), + ('status', models.CharField(max_length=50)), + ('aws_region', models.CharField(blank=True, max_length=50, null=True)), + ('slug', models.SlugField(blank=True, max_length=255, unique=True)), + ('discovered_at', models.DateTimeField(auto_now_add=True)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='log_sources', to='aws.awsaccount')), + ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_log_sources', to='case.case')), + ], + ), migrations.CreateModel( name='AWSResource', fields=[ @@ -37,7 +52,7 @@ class Migration(migrations.Migration): ('resource_type', models.CharField(max_length=100)), ('resource_name', models.CharField(blank=True, max_length=200, null=True)), ('resource_details', models.JSONField(blank=True, null=True)), - ('aws_region', models.CharField(default='us-east-1', max_length=50)), + ('aws_region', models.CharField(blank=True, max_length=50, null=True)), ('slug', models.SlugField(blank=True, max_length=255, unique=True)), ('discovered_at', models.DateTimeField(auto_now_add=True)), ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='resources', to='aws.awsaccount')), diff --git a/apps/aws/migrations/0002_alter_awsresource_aws_region.py b/apps/aws/migrations/0002_alter_awsresource_aws_region.py deleted file mode 100644 index a7ef3a7..0000000 --- a/apps/aws/migrations/0002_alter_awsresource_aws_region.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 00:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('aws', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='awsresource', - name='aws_region', - field=models.CharField(blank=True, max_length=50, null=True), - ), - ] diff --git a/apps/aws/migrations/0003_awslogsource.py b/apps/aws/migrations/0003_awslogsource.py deleted file mode 100644 index 2b3b039..0000000 --- a/apps/aws/migrations/0003_awslogsource.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 04:16 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('aws', '0002_alter_awsresource_aws_region'), - ('case', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='AWSLogSource', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('service_name', models.CharField(max_length=100)), - ('log_name', models.CharField(max_length=255)), - ('log_details', models.JSONField(blank=True, null=True)), - ('status', models.CharField(max_length=50)), - ('aws_region', models.CharField(blank=True, max_length=50, null=True)), - ('slug', models.SlugField(blank=True, max_length=255, unique=True)), - ('discovered_at', models.DateTimeField(auto_now_add=True)), - ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='log_sources', to='aws.awsaccount')), - ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_log_sources', to='case.case')), - ], - ), - ] diff --git a/apps/aws/urls.py b/apps/aws/urls.py index 521336f..44e1a68 100644 --- a/apps/aws/urls.py +++ b/apps/aws/urls.py @@ -11,11 +11,12 @@ path('accounts//delete/', views.delete_account, name='delete_account'), path('accounts//pull-resources/', views.pull_resources_view, name='pull_aws_resources'), path('resources//details/', views.aws_resource_details, name='aws_resource_details'), - path('accounts//account-details/', views.account_details, name='account_details'), + path('accounts//account-resources/', views.account_resources, name='account_resources'), path('logsource///details/', views.aws_logsource_details, name='aws_logsource_details'), path('fetch-management-events//', views.trigger_management_event_fetch, name='fetch_management_events'), path('browse-s3-structure/', views.browse_s3_structure, name='browse_s3_structure'), path('fetch-logs//', views.fetch_cloudtrail_logs, name='fetch_cloudtrail_logs'), + path('accounts//logs/', views.normalized_logs_view, name='normalized_logs'), ] diff --git a/apps/aws/utils.py b/apps/aws/utils.py index d782518..c2634a3 100644 --- a/apps/aws/utils.py +++ b/apps/aws/utils.py @@ -3,14 +3,19 @@ from apps.data.models import NormalizedLog from apps.case.models import Case from datetime import datetime, timedelta -import datetime, gzip +from django.utils import timezone +import gzip from botocore.exceptions import EndpointConnectionError, ClientError import logging from django.db import transaction -from django.utils import timezone +from datetime import timezone +from django.utils.timezone import make_aware +from datetime import datetime, timezone + import json + logger = logging.getLogger(__name__) # Validate AWS credentials by calling the STS GetCallerIdentity API. @@ -339,98 +344,98 @@ def fetch_guardduty_detectors(): # this function extracts logs from a s3 bucket based on the users selected prefix def fetch_and_normalize_cloudtrail_logs(account_id, resource_id, prefix, start_date, end_date, case_id): try: - account = AWSAccount.objects.get(account_id=account_id) + aws_account = AWSAccount.objects.get(account_id=account_id) + case = Case.objects.get(id=case_id) resource = AWSResource.objects.get(id=resource_id) - except (AWSAccount.DoesNotExist, AWSResource.DoesNotExist): - print("DEBUG: Account or Resource not found.") + except (AWSAccount.DoesNotExist, AWSResource.DoesNotExist, Case.DoesNotExist): + print("DEBUG: Account, Resource, or Case not found.") return session = boto3.Session( - aws_access_key_id=account.aws_access_key, - aws_secret_access_key=account.aws_secret_key, - region_name=resource.aws_region or account.default_region + aws_access_key_id=aws_account.aws_access_key, + aws_secret_access_key=aws_account.aws_secret_key, + region_name=resource.aws_region or aws_account.aws_region ) s3 = session.client("s3") bucket_name = resource.resource_name or resource.resource_id - # Convert date strings to date objects - start_date_obj = datetime.datetime.strptime(start_date, "%Y-%m-%d").date() - end_date_obj = datetime.datetime.strptime(end_date, "%Y-%m-%d").date() + start_date_obj = datetime.strptime(start_date, "%Y-%m-%d").date() + end_date_obj = datetime.strptime(end_date, "%Y-%m-%d").date() - # Ensure prefix ends with '/' if prefix and not prefix.endswith("/"): prefix += "/" current_date = start_date_obj - while current_date <= end_date_obj: - date_folder = f"{current_date.year}/{current_date.strftime('%m')}/{current_date.strftime('%d')}/" - final_prefix = prefix + date_folder - - print(f"DEBUG: Checking prefix '{final_prefix}' in bucket '{bucket_name}'") - - paginator = s3.get_paginator("list_objects_v2") - page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=final_prefix) - - any_objects_found = False - for page in page_iterator: - contents = page.get("Contents", []) - if not contents: - print(f"DEBUG: No objects in prefix '{final_prefix}' on this page.") - continue + with transaction.atomic(): + while current_date <= end_date_obj: + date_folder = f"{current_date.year}/{current_date.strftime('%m')}/{current_date.strftime('%d')}/" + final_prefix = prefix + date_folder - any_objects_found = True + print(f"DEBUG: Checking prefix '{final_prefix}' in bucket '{bucket_name}'") - for obj in contents: - key = obj["Key"] - try: - resp = s3.get_object(Bucket=bucket_name, Key=key) - except Exception as e: - print(f"DEBUG: Error fetching object '{key}' - {e}") - continue + paginator = s3.get_paginator("list_objects_v2") + page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=final_prefix) - raw_body = resp["Body"].read() - # Decompress if .gz - if key.endswith(".gz"): + for page in page_iterator: + contents = page.get("Contents", []) + for obj in contents: + key = obj["Key"] try: - file_data = gzip.decompress(raw_body).decode("utf-8") - except OSError: - print(f"DEBUG: Failed to decompress '{key}'. Skipping.") + resp = s3.get_object(Bucket=bucket_name, Key=key) + except Exception as e: + print(f"DEBUG: Error fetching object '{key}' - {e}") continue - else: - file_data = raw_body.decode("utf-8") - - # Parse CloudTrail JSON - try: - records_json = json.loads(file_data) - records = records_json.get("Records", []) - except json.JSONDecodeError: - print(f"DEBUG: JSON decode error in '{key}'. Skipping.") - continue - - print(f"DEBUG: Found {len(records)} log records in file '{key}'") - for record in records: - NormalizedLog.objects.create( - case_id=case_id, - log_id=record.get("eventID", ""), - log_source="aws", - log_type="CloudTrail", - event_name=record.get("eventName", ""), - event_time=record.get("eventTime", timezone.now()), - user_identity=record.get("userIdentity", {}), - ip_address=record.get("sourceIPAddress"), - resources=record.get("resources", []), - raw_data=record, - ) - - if not any_objects_found: - print(f"DEBUG: No objects found at all for prefix '{final_prefix}'") - - current_date += datetime.timedelta(days=1) + + raw_body = resp["Body"].read() + if key.endswith(".gz"): + try: + file_data = gzip.decompress(raw_body).decode("utf-8") + except OSError: + print(f"DEBUG: Failed to decompress '{key}'. Skipping.") + continue + else: + file_data = raw_body.decode("utf-8") + + try: + records_json = json.loads(file_data) + records = records_json.get("Records", []) + except json.JSONDecodeError: + print(f"DEBUG: JSON decode error in '{key}'. Skipping.") + continue + + for record in records: + try: + raw_event = json.loads(json.dumps(record, default=str)) + event_time = raw_event.get('eventTime') + if event_time: + event_time = datetime.strptime(event_time, "%Y-%m-%dT%H:%M:%SZ") + event_time = make_aware(event_time, timezone.utc) + + log_data = { + 'case': case, + 'log_id': raw_event.get('eventID'), + 'log_source': 'aws', + 'log_type': 'CloudTrail', + 'event_name': raw_event.get('eventName', 'Unknown'), + 'event_time': event_time, + 'user_identity': raw_event.get('userIdentity', {}).get('userName') or raw_event.get('userIdentity', {}).get('invokedBy', 'Unknown'), + 'ip_address': raw_event.get('sourceIPAddress', None), + 'resources': raw_event.get('resources', []), + 'raw_data': raw_event, + 'extra_data': {}, + 'aws_account': aws_account, + } + + NormalizedLog.objects.create(**log_data) + except Exception as e: + print(f"DEBUG: Error saving log for eventID {raw_event.get('eventID')}: {e}") + continue + + current_date += timedelta(days=1) print("DEBUG: Finished fetching logs.") def fetch_management_event_history(account_id, case_id): - aws_account = AWSAccount.objects.get(account_id=account_id) case = Case.objects.get(id=case_id) @@ -441,36 +446,51 @@ def fetch_management_event_history(account_id, case_id): ) client = session.client('cloudtrail') - # Paginator for management event history paginator = client.get_paginator('lookup_events') - with transaction.atomic(): - for page in paginator.paginate(): - for event in page.get('Events', []): - # Ensure event data is JSON serializable - raw_event = json.loads(json.dumps(event, default=str)) - + for page in paginator.paginate(): + for event in page.get('Events', []): + try: + # Handle event time: check if it's already a datetime object + event_time = event.get('EventTime') + if isinstance(event_time, str): + event_time = datetime.strptime(event_time, "%Y-%m-%dT%H:%M:%SZ") + event_time = make_aware(event_time, utc) + + # Extract user identity or service name + user_identity_value = ( + event.get('Username') or + event.get('userIdentity', {}).get('userName') or + event.get('userIdentity', {}).get('invokedBy') or + 'Unknown' + ) + + # Extract IP address from CloudTrailEvent + cloudtrail_event = json.loads(event.get('CloudTrailEvent', '{}')) + ip_address = cloudtrail_event.get('sourceIPAddress', 'Unknown') + + # Prepare log data log_data = { - 'case_id': case_id, - 'log_id': raw_event.get('EventId'), + 'case': case, + 'log_id': event.get('EventId'), 'log_source': 'aws', - 'log_type': raw_event.get('EventSource'), - 'event_name': raw_event.get('EventName'), - 'event_time': raw_event.get('EventTime'), - 'user_identity': raw_event.get('Username', {}), - 'ip_address': raw_event.get('SourceIPAddress'), - 'resources': raw_event.get('Resources', []), - 'raw_data': raw_event, # Ensure all fields are JSON serializable - 'extra_data': {}, # Add additional metadata if needed + 'log_type': event.get('EventSource', 'Unknown'), + 'event_name': event.get('EventName', 'Unknown'), + 'event_time': event_time, + 'user_identity': user_identity_value, + 'ip_address': ip_address, + 'resources': event.get('Resources', []), + 'raw_data': event, + 'extra_data': {}, + 'aws_account': aws_account, } - # Normalize and save the log - normalized_log = NormalizedLog.objects.create(**log_data) + # Save log + NormalizedLog.objects.create(**log_data) + + except Exception as e: + # Log the error and continue with the next event + print(f"DEBUG: Error saving event {event.get('EventId', 'Unknown')} - {e}") - # Link to resources if applicable - for resource in raw_event.get('Resources', []): - aws_resource = AWSResource.objects.filter(resource_name=resource.get('ResourceName')).first() - if aws_resource: - normalized_log.aws_resources.add(aws_resource) diff --git a/apps/aws/views.py b/apps/aws/views.py index 664f52b..007e38f 100644 --- a/apps/aws/views.py +++ b/apps/aws/views.py @@ -7,10 +7,14 @@ from .utils import validate_aws_credentials from django.contrib import messages from .tasks import pull_aws_resources_task, fetch_management_history_task, fetch_normalize_cloudtrail_logs_task -from datetime import datetime +from datetime import datetime, timedelta from django.utils import timezone from django.http import JsonResponse from django.utils.timezone import make_aware +from apps.data.models import NormalizedLog +from django.db.models import Count, Value +from django.db.models.functions import Coalesce + import logging @@ -45,7 +49,7 @@ def connect_aws(request, slug): messages.error(request, f"AWS account saved, but validation failed: {error_message}") # Redirect to connected accounts page - return redirect('case:list_connected_accounts', slug=case.slug) + return redirect('case:case_detail', slug=case.slug) else: form = AWSAccountForm() @@ -59,7 +63,7 @@ def edit_account(request, account_id): form = AWSAccountForm(request.POST, instance=account) if form.is_valid(): form.save() - return redirect('case:list_connected_accounts', slug=account.case.slug) + return redirect('case:case_detail', slug=account.case.slug) else: form = AWSAccountForm(instance=account) @@ -71,7 +75,7 @@ def delete_account(request, account_id): account = get_object_or_404(AWSAccount, id=account_id) slug = account.case.slug # Save the slug for redirection account.delete() - return redirect('case:list_connected_accounts', slug=slug) + return redirect('case:case_detail', slug=slug) #This is the trigger for getting the resources of the AWS account. @@ -82,13 +86,13 @@ def pull_resources_view(request, account_id): if not aws_account.validated: messages.error(request, "Cannot pull resources because the AWS account credentials are not validated.") - return redirect('case:list_connected_accounts', slug=aws_account.case.slug) + return redirect('case:case_detail', slug=aws_account.case.slug) # Trigger background task pull_aws_resources_task.delay(account_id) - messages.info(request, "Resource pulling has started. This may take a few minutes.") + messages.info(request, "Resource pulling has started. Refresh the page after after a few minutes to see the results.") - return redirect('case:list_connected_accounts', slug=aws_account.case.slug) + return redirect('aws:account_resources', account_id=aws_account.account_id) # Open a modal to show the details of the resources @login_required @@ -101,8 +105,9 @@ def aws_resource_details(request, resource_id): # this renders both the aws resources and logging sources into one page @login_required -def account_details(request, account_id): - aws_account = get_object_or_404(AWSAccount, id=account_id) +def account_resources(request, account_id): + aws_account = get_object_or_404(AWSAccount, account_id=account_id) + case = aws_account.case # Group resources by their type resources = AWSResource.objects.filter(account=aws_account).order_by('resource_type', 'resource_name') @@ -125,11 +130,53 @@ def account_details(request, account_id): context = { 'aws_account': aws_account, + 'case': case, 'grouped_resources': grouped_resources, 'grouped_log_sources': grouped_log_sources, 'error_messages': error_messages, } - return render(request, 'aws/account_details.html', context) + return render(request, 'aws/account_resources.html', context) + + +@login_required +def normalized_logs_view(request, account_id): + aws_account = get_object_or_404(AWSAccount, account_id=account_id) + + # Default date filter: Last day + end_date = datetime.now() + start_date = request.GET.get("start_date", (end_date - timedelta(days=1)).strftime("%Y-%m-%d")) + end_date = request.GET.get("end_date", end_date.strftime("%Y-%m-%d")) + + # Convert to date objects + start_date = datetime.strptime(start_date, "%Y-%m-%d").date() + end_date = datetime.strptime(end_date, "%Y-%m-%d").date() + + # Filter logs for the specific AWSAccount within the date range + logs = NormalizedLog.objects.filter( + aws_account=aws_account, + event_time__date__gte=start_date, + event_time__date__lte=end_date + ) + + # Aggregate top 10 users + top_users = logs.values('user_identity').annotate(count=Count('user_identity')).order_by('-count')[:10] + + # Aggregate top 10 IPs + top_ips = logs.values('ip_address').annotate(count=Count('ip_address')).order_by('-count')[:10] + + # Aggregate top 10 events + top_events = logs.values('event_name').annotate(count=Count('event_name')).order_by('-count')[:10] + + context = { + "aws_account": aws_account, + "logs": logs, + "top_users": top_users, + "top_ips": top_ips, + "top_events": top_events, + "start_date": start_date, + "end_date": end_date, + } + return render(request, "aws/normalized_logs.html", context) @login_required def aws_logsource_details(request, slug): @@ -183,7 +230,7 @@ def fetch_cloudtrail_logs(request, account_id): if resource.account.account_id != aws_account.account_id: messages.error(request, "Selected bucket is not linked to this AWS account.") - return redirect("case:list_connected_accounts", slug=resource.case.slug) + return redirect("case:case_detail", slug=resource.case.slug) fetch_normalize_cloudtrail_logs_task.delay( account_id=aws_account.account_id, @@ -194,7 +241,7 @@ def fetch_cloudtrail_logs(request, account_id): case_id=resource.case.id ) messages.success(request, "CloudTrail log fetching has been queued.") - return redirect("case:list_connected_accounts", slug=resource.case.slug) + return redirect("aws:normalized_logs", account_id=aws_account.account_id) else: form = FetchCloudTrailLogsForm() @@ -210,5 +257,5 @@ def trigger_management_event_fetch(request, account_id): messages.success(request, "Management event history is being fetched.") logger.info(f"Task queued for AWS account {account_id}") - return redirect('case:list_connected_accounts', slug=aws_account.case.slug) + return redirect("aws:normalized_logs", account_id=aws_account.account_id) diff --git a/apps/case/urls.py b/apps/case/urls.py index 7b881d8..92fbcfa 100644 --- a/apps/case/urls.py +++ b/apps/case/urls.py @@ -10,6 +10,5 @@ path('/', views.case_detail, name='case_detail'), path('/edit/', views.edit_case, name='edit_case'), path('/connect/', views.connect_client, name='connect_client'), - path('/connected-accounts/', views.list_connected_accounts, name='list_connected_accounts'), ] diff --git a/apps/case/views.py b/apps/case/views.py index 125ae78..a4c58bc 100644 --- a/apps/case/views.py +++ b/apps/case/views.py @@ -23,8 +23,21 @@ def create_case(request): # This is used to view the details of a case @login_required def case_detail(request, slug): - case = Case.objects.get(slug=slug) - return render(request, "case/case_detail.html", {"case": case}) + case = get_object_or_404(Case, slug=slug) + + # AWS accounts linked to the case + aws_accounts = AWSAccount.objects.filter(case=case) + + # Add GCP and Azure placeholders + gcp_placeholder = True + azure_placeholder = True + + return render(request, "case/case_detail.html", { + "case": case, + "aws_accounts": aws_accounts, + "gcp_placeholder": gcp_placeholder, + "azure_placeholder": azure_placeholder, + }) # this is used to edit the details of a case @login_required @@ -50,20 +63,3 @@ def edit_case(request, slug): def connect_client(request, slug): case = get_object_or_404(Case, slug=slug) return render(request, 'case/connect_client.html', {'case': case}) - -@login_required -def list_connected_accounts(request, slug): - case = get_object_or_404(Case, slug=slug) - - # AWS accounts linked to the case - aws_accounts = AWSAccount.objects.filter(case=case) - - # Add GCP and azure later - - return render(request, 'case/list_connected_accounts.html', { - 'case': case, - 'aws_accounts': aws_accounts, - # GCP and Azure placeholders - 'gcp_placeholder': True, - 'azure_placeholder': True, - }) \ No newline at end of file diff --git a/apps/data/migrations/0001_initial.py b/apps/data/migrations/0001_initial.py index 6eb1dc2..623b6ad 100644 --- a/apps/data/migrations/0001_initial.py +++ b/apps/data/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 5.1.3 on 2024-12-25 11:56 +# Generated by Django 5.1.3 on 2025-01-20 04:31 +import django.contrib.postgres.indexes import django.db.models.deletion from django.db import migrations, models @@ -18,17 +19,22 @@ class Migration(migrations.Migration): name='NormalizedLog', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('log_source', models.CharField(choices=[('aws', 'Amazon Web Services'), ('gcp', 'Google Cloud Platform'), ('azure', 'Microsoft Azure')], max_length=50)), - ('log_type', models.CharField(max_length=100)), - ('event_name', models.CharField(max_length=255)), - ('event_time', models.DateTimeField()), - ('user_identity', models.JSONField(blank=True, null=True)), - ('resources', models.JSONField(blank=True, null=True)), - ('raw_data', models.JSONField()), - ('extra_data', models.JSONField(blank=True, null=True)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('aws_resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='normalized_logs', to='aws.awsresource')), + ('log_id', models.CharField(blank=True, max_length=255, null=True)), + ('log_source', models.CharField(choices=[('aws', 'Amazon Web Services'), ('gcp', 'Google Cloud Platform'), ('azure', 'Microsoft Azure')], db_index=True, max_length=50)), + ('log_type', models.CharField(db_index=True, max_length=100)), + ('event_name', models.CharField(db_index=True, max_length=255)), + ('event_time', models.DateTimeField(db_index=True)), + ('user_identity', models.CharField(blank=True, max_length=255, null=True)), + ('ip_address', models.GenericIPAddressField(blank=True, db_index=True, null=True)), + ('resources', models.TextField(blank=True, null=True)), + ('raw_data', models.TextField()), + ('extra_data', models.TextField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('aws_account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='normalized_logs', to='aws.awsaccount')), ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='normalized_logs', to='case.case')), ], + options={ + 'indexes': [django.contrib.postgres.indexes.BTreeIndex(fields=['log_source'], name='data_normal_log_sou_bee0df_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['log_type'], name='data_normal_log_typ_a9e795_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['event_name'], name='data_normal_event_n_8b8bea_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['event_time'], name='data_normal_event_t_c05244_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['ip_address'], name='data_normal_ip_addr_3df897_btree')], + }, ), ] diff --git a/apps/data/migrations/0002_remove_normalizedlog_aws_resource_and_more.py b/apps/data/migrations/0002_remove_normalizedlog_aws_resource_and_more.py deleted file mode 100644 index 61546fd..0000000 --- a/apps/data/migrations/0002_remove_normalizedlog_aws_resource_and_more.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 02:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('aws', '0002_alter_awsresource_aws_region'), - ('data', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='normalizedlog', - name='aws_resource', - ), - migrations.AddField( - model_name='normalizedlog', - name='aws_resources', - field=models.ManyToManyField(related_name='normalized_logs', to='aws.awsresource'), - ), - migrations.AddField( - model_name='normalizedlog', - name='ip_address', - field=models.GenericIPAddressField(blank=True, null=True), - ), - ] diff --git a/apps/data/migrations/0003_normalizedlog_log_id.py b/apps/data/migrations/0003_normalizedlog_log_id.py deleted file mode 100644 index 99412be..0000000 --- a/apps/data/migrations/0003_normalizedlog_log_id.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 02:30 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('data', '0002_remove_normalizedlog_aws_resource_and_more'), - ] - - operations = [ - migrations.AddField( - model_name='normalizedlog', - name='log_id', - field=models.CharField(blank=True, max_length=255, null=True), - ), - ] diff --git a/apps/data/models.py b/apps/data/models.py index da2dc50..495d140 100644 --- a/apps/data/models.py +++ b/apps/data/models.py @@ -1,10 +1,8 @@ from django.db import models -from apps.aws.models import AWSResource +from apps.aws.models import AWSAccount from apps.case.models import Case +from django.contrib.postgres.indexes import BTreeIndex -# Create your models here. - -# This stores all the logs in a normalised fashion. All logs from all services are normalized to allow for standard queries class NormalizedLog(models.Model): SOURCE = [ ("aws", "Amazon Web Services"), @@ -13,24 +11,31 @@ class NormalizedLog(models.Model): ] case = models.ForeignKey(Case, on_delete=models.CASCADE, related_name='normalized_logs') - log_id = models.CharField(max_length=255, blank=True, null=True) # Log id from the service - log_source = models.CharField(max_length=50, choices=SOURCE) - log_type = models.CharField(max_length=100) - event_name = models.CharField(max_length=255) - event_time = models.DateTimeField() - user_identity = models.JSONField(blank=True, null=True) - ip_address = models.GenericIPAddressField(blank=True, null=True) - resources = models.JSONField(blank=True, null=True) - raw_data = models.JSONField() - extra_data = models.JSONField(blank=True, null=True) + log_id = models.CharField(max_length=255, blank=True, null=True) # Log ID from the service + log_source = models.CharField(max_length=50, choices=SOURCE, db_index=True) # Indexed + log_type = models.CharField(max_length=100, db_index=True) # Indexed + event_name = models.CharField(max_length=255, db_index=True) # Indexed + event_time = models.DateTimeField(db_index=True) # Indexed for date filtering + user_identity = models.CharField(max_length=255, blank=True, null=True) # Stores username or user info + ip_address = models.GenericIPAddressField(blank=True, null=True, db_index=True) # Indexed + resources = models.TextField(blank=True, null=True) # Serialized list of resources as text + raw_data = models.TextField() # Serialized JSON as text + extra_data = models.TextField(blank=True, null=True) # Serialized additional metadata as text # Relationships - aws_resources = models.ManyToManyField(AWSResource, related_name='normalized_logs') + aws_account = models.ForeignKey(AWSAccount, on_delete=models.CASCADE, related_name='normalized_logs') # Utility - created_at = models.DateTimeField(auto_now_add=True) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) # Indexed for sorting by creation time def __str__(self): return f"{self.event_name} ({self.log_source}) - Case {self.case.name}" - + class Meta: + indexes = [ + BTreeIndex(fields=["log_source"]), + BTreeIndex(fields=["log_type"]), + BTreeIndex(fields=["event_name"]), + BTreeIndex(fields=["event_time"]), + BTreeIndex(fields=["ip_address"]), + ] diff --git a/templates/account/login.html b/templates/account/login.html index 5f56a9c..c71d620 100644 --- a/templates/account/login.html +++ b/templates/account/login.html @@ -12,8 +12,7 @@

{% translate "Sign In" %}

{% render_text_input form.login %} {% render_text_input form.password %} - - {% include 'account/components/social/social_buttons.html' %} + } {% if LOGIN_BY_CODE_ENABLED %}

diff --git a/templates/aws/account_details.html b/templates/aws/account_resources.html similarity index 64% rename from templates/aws/account_details.html rename to templates/aws/account_resources.html index 11bdef5..dd07872 100644 --- a/templates/aws/account_details.html +++ b/templates/aws/account_resources.html @@ -3,14 +3,37 @@ {% load static %} {% block app %} -

+
+ + + -

Account Details for: {{ aws_account.account_id }}

+
+

Account Details for: {{ aws_account.account_id }}

+ + Pull Resources + +

AWS Resources

+ {% if grouped_resources %} {% for resource_type, resources in grouped_resources.items %}

{{ resource_type }}

@@ -36,6 +59,14 @@

{% endfor %} + {% else %} +
+

No resources found. Would you like to pull resources?

+ + Pull Resources + +
+ {% endif %} @@ -46,6 +77,7 @@

AWS Log Sources

{{ message }}
{% endfor %} + {% if grouped_log_sources %} {% for service_name, log_sources in grouped_log_sources.items %}

{{ service_name }}

@@ -68,6 +100,11 @@

{{ log_source.log_name }}

{% endfor %} + {% else %} +
+

No log sources found.

+
+ {% endif %} - + {% endblock %} diff --git a/templates/aws/normalized_logs.html b/templates/aws/normalized_logs.html new file mode 100644 index 0000000..b175ecd --- /dev/null +++ b/templates/aws/normalized_logs.html @@ -0,0 +1,118 @@ +{% extends "web/app/app_base.html" %} +{% load i18n %} +{% load static %} + +{% block app %} +
+ + + + +
+

Logs for Account: {{ aws_account.account_id }}

+ +
+ + +
+
+
+ + +
+
+ + +
+
+ +
+
+
+ + +
+ +
+
+
+

Top 10 Users

+
    + {% for user in top_users %} +
  • + {{ user.user_identity|default:"Unknown User" }} + {{ user.count }} +
    • + {% empty %} +
  • + No users found. +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+

Top 10 IPs

+
    + {% for ip in top_ips %} +
  • + {{ ip.ip_address|default:"Unknown IP" }} + {{ ip.count }} +
    • + {% empty %} +
  • + No IPs found. +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+

Top 10 Events

+
    + {% for event in top_events %} +
  • + {{ event.event_name|default:"Unknown Event" }} + {{ event.count }} +
    • + {% empty %} +
  • + No events found. +
    • + {% endfor %} +
+
+
+
+
+
+{% endblock %} diff --git a/templates/case/case_detail.html b/templates/case/case_detail.html index b1fea58..5091e33 100644 --- a/templates/case/case_detail.html +++ b/templates/case/case_detail.html @@ -3,24 +3,92 @@ {% load static %} {% block app %} -
-

{{ case.name }}

-

{{ case.description }}

-

Status: {{ case.status }}

-

Created by: {{ case.created_by.username }}

-

Created at: {{ case.created_at }}

- -
- Edit Case +
+ + + + +
+ +
+

{{ case.name }}

+

Status: {{ case.status }}

+

Created At: {{ case.created_at|date:"Y-m-d H:i" }}

+

Case ID: {{ case.uuid }}

+

{{ case.description }}

-
- Connect Client + + +
+ Edit Case + Connect Client
-
- - View Connections - +
+ + +

Connected Accounts

+ {% if aws_accounts %} +
+ {% for account in aws_accounts %} +
+
+ +
+
+ AWS Account: {{ account.account_id }} +
+

+ Region: {{ account.aws_region }} | + Added By: {{ account.added_by.username }} | + Added At: {{ account.added_at|date:"Y-m-d H:i" }} +

+ {% if account.validated %} + Validated + {% else %} + Not Validated + {% endif %} +
+ + +
+ Edit + + + Resources + + + Logging + +
+
-
+ {% endfor %} +
+ {% else %} +

No accounts connected yet. Use "Connect Client" to add accounts.

+ {% endif %} + + +

GCP Accounts

+ {% if gcp_placeholder %} +

GCP integration coming soon.

+ {% endif %} + +

Azure Accounts

+ {% if azure_placeholder %} +

Azure integration coming soon.

+ {% endif %} +
{% endblock %} diff --git a/templates/case/list_connected_accounts.html b/templates/case/list_connected_accounts.html deleted file mode 100644 index f737d93..0000000 --- a/templates/case/list_connected_accounts.html +++ /dev/null @@ -1,89 +0,0 @@ -{% extends "web/app/app_base.html" %} -{% load i18n %} -{% load static %} -{% block app %} - -
-

Connected Accounts for Case: {{ case.name }}

- - -
- Return to Case -
- -
- Connect Another Account -
- - -

AWS Accounts

- {% if aws_accounts %} -
- - - - - - - - - - - - {% for account in aws_accounts %} - - - - - - - - - {% endfor %} - -
Account IDRegionAdded ByAdded AtActions
- - {{ account.account_id }} - - {{ account.aws_region }}{{ account.added_by.username }}{{ account.added_at|date:"Y-m-d H:i" }} - {% if account.validated %} - Validated - {% else %} - Not Validated - {% endif %} - - Edit - - - Pull Resources - - - Get CloudTrail Logs - - - Get Management Logs (90 days) - -
-
- {% else %} -

No AWS accounts connected yet.

- {% endif %} - - -

GCP Accounts

- {% if gcp_placeholder %} -

GCP integration coming soon.

- {% endif %} - - -

Azure Accounts

- {% if azure_placeholder %} -

Azure integration coming soon.

- {% endif %} -
- -{% endblock %} diff --git a/templates/web/app/app_base.html b/templates/web/app/app_base.html index 4a55803..b795c27 100644 --- a/templates/web/app/app_base.html +++ b/templates/web/app/app_base.html @@ -3,14 +3,9 @@ {% block body %} {% block notifications %} {% endblock %} -
-
-
- {% block sidebar-nav %} - {% include "web/components/app_nav.html" %} - {% endblock %} -
-
+
+
+
{% block app %} {% endblock %}
diff --git a/templates/web/app_home.html b/templates/web/app_home.html index 1657af4..82314c0 100644 --- a/templates/web/app_home.html +++ b/templates/web/app_home.html @@ -4,39 +4,45 @@ {% block app %}
-
-

Your Cases

- -
- Create New Case +
+ +
+

Your Cases

+ Create New Case
+ {% if cases %} - - - - - - - - +
+
NameStatusCreated AtActions
+ + + + + + + - {% for case in cases %} - - - - - - - {% endfor %} + {% for case in cases %} + + + + + + + {% endfor %} -
NameStatusCreated AtActions
{{ case.name }}{{ case.status }}{{ case.created_at|date:"Y-m-d H:i" }} - View Details -
{{ case.name }}{{ case.status }}{{ case.created_at|date:"Y-m-d H:i" }} + View Case +
+ +
{% else %} -

You have no cases yet. Create a new one to get started.

+ {% endif %} -
+
+ {% endblock %} diff --git a/templates/web/components/footer.html b/templates/web/components/footer.html index 97cdef6..a6a240d 100644 --- a/templates/web/components/footer.html +++ b/templates/web/components/footer.html @@ -1,11 +1,9 @@ {% load i18n %}
-

{{project_meta.NAME}} — {% translate "Copyright" %} 2024

+

Scope Forensics

- + diff --git a/templates/web/components/top_nav.html b/templates/web/components/top_nav.html index 8b0811d..23a70b1 100644 --- a/templates/web/components/top_nav.html +++ b/templates/web/components/top_nav.html @@ -2,7 +2,7 @@