diff --git a/apps/aws/admin.py b/apps/aws/admin.py index e369ace..a5f7083 100644 --- a/apps/aws/admin.py +++ b/apps/aws/admin.py @@ -1,10 +1,10 @@ from django.contrib import admin -from .models import AWSAccount, AWSResource, AWSLogSource +from .models import AWSAccount, AWSResource, AWSLogSource, AWSCredential # Register your models here. admin.site.register(AWSAccount) admin.site.register(AWSResource) admin.site.register(AWSLogSource) - +admin.site.register(AWSCredential) diff --git a/apps/aws/migrations/0001_initial.py b/apps/aws/migrations/0001_initial.py index 7be47ef..a6293f7 100644 --- a/apps/aws/migrations/0001_initial.py +++ b/apps/aws/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.3 on 2024-12-25 11:56 +# Generated by Django 5.1.3 on 2025-01-22 05:31 import django.db.models.deletion from django.conf import settings @@ -29,6 +29,21 @@ class Migration(migrations.Migration): ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_accounts', to='case.case')), ], ), + migrations.CreateModel( + name='AWSLogSource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('service_name', models.CharField(max_length=100)), + ('log_name', models.CharField(max_length=255)), + ('log_details', models.JSONField(blank=True, null=True)), + ('status', models.CharField(max_length=50)), + ('aws_region', models.CharField(blank=True, max_length=50, null=True)), + ('slug', models.SlugField(blank=True, max_length=255, unique=True)), + ('discovered_at', models.DateTimeField(auto_now_add=True)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='log_sources', to='aws.awsaccount')), + ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_log_sources', to='case.case')), + ], + ), migrations.CreateModel( name='AWSResource', fields=[ @@ -37,11 +52,46 @@ class Migration(migrations.Migration): ('resource_type', models.CharField(max_length=100)), ('resource_name', models.CharField(blank=True, max_length=200, null=True)), ('resource_details', models.JSONField(blank=True, null=True)), - ('aws_region', models.CharField(default='us-east-1', max_length=50)), + ('aws_region', models.CharField(blank=True, max_length=50, null=True)), ('slug', models.SlugField(blank=True, max_length=255, unique=True)), ('discovered_at', models.DateTimeField(auto_now_add=True)), ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='resources', to='aws.awsaccount')), ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_resources', to='case.case')), ], ), + migrations.CreateModel( + name='AWSCredential', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('user', models.CharField(max_length=300)), + ('user_arn', models.CharField(max_length=300)), + ('user_creation_time', models.DateTimeField(blank=True, null=True)), + ('password_enabled', models.BooleanField(default=False)), + ('password_last_used', models.DateTimeField(blank=True, null=True)), + ('password_last_changed', models.DateTimeField(blank=True, null=True)), + ('password_next_rotation_date', models.DateTimeField(blank=True, null=True)), + ('mfa_active', models.BooleanField(default=False)), + ('access_key_1_active', models.BooleanField(default=False)), + ('access_key_1_last_rotated', models.DateTimeField(blank=True, null=True)), + ('access_key_1_last_used_date', models.DateTimeField(blank=True, null=True)), + ('access_key_1_last_used_region', models.CharField(blank=True, max_length=300, null=True)), + ('access_key_1_last_used_service', models.CharField(blank=True, max_length=300, null=True)), + ('access_key_2_active', models.BooleanField(default=False)), + ('access_key_2_last_rotated', models.DateTimeField(blank=True, null=True)), + ('access_key_2_last_used_date', models.DateTimeField(blank=True, null=True)), + ('access_key_2_last_used_region', models.CharField(blank=True, max_length=300, null=True)), + ('access_key_2_last_used_service', models.CharField(blank=True, max_length=300, null=True)), + ('cert_1_active', models.BooleanField(default=False)), + ('cert_1_last_rotated', models.DateTimeField(blank=True, null=True)), + ('cert_2_active', models.BooleanField(default=False)), + ('cert_2_last_rotated', models.DateTimeField(blank=True, null=True)), + ('slug', models.SlugField(blank=True, max_length=255, unique=True)), + ('discovered_at', models.DateTimeField(auto_now_add=True)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='credentials', to='aws.awsaccount')), + ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_credentials', to='case.case')), + ], + options={ + 'unique_together': {('account', 'user')}, + }, + ), ] diff --git a/apps/aws/migrations/0002_alter_awsresource_aws_region.py b/apps/aws/migrations/0002_alter_awsresource_aws_region.py deleted file mode 100644 index a7ef3a7..0000000 --- a/apps/aws/migrations/0002_alter_awsresource_aws_region.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 00:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('aws', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='awsresource', - name='aws_region', - field=models.CharField(blank=True, max_length=50, null=True), - ), - ] diff --git a/apps/aws/migrations/0002_awscredential_tags_awslogsource_tags_and_more.py b/apps/aws/migrations/0002_awscredential_tags_awslogsource_tags_and_more.py new file mode 100644 index 0000000..4291972 --- /dev/null +++ b/apps/aws/migrations/0002_awscredential_tags_awslogsource_tags_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 5.1.3 on 2025-01-24 03:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('aws', '0001_initial'), + ('data', '0003_normalizedlog_tags'), + ] + + operations = [ + migrations.AddField( + model_name='awscredential', + name='tags', + field=models.ManyToManyField(related_name='aws_credential', to='data.tag'), + ), + migrations.AddField( + model_name='awslogsource', + name='tags', + field=models.ManyToManyField(related_name='aws_log_source', to='data.tag'), + ), + migrations.AddField( + model_name='awsresource', + name='tags', + field=models.ManyToManyField(related_name='aws_resource', to='data.tag'), + ), + ] diff --git a/apps/aws/migrations/0003_awslogsource.py b/apps/aws/migrations/0003_awslogsource.py deleted file mode 100644 index 2b3b039..0000000 --- a/apps/aws/migrations/0003_awslogsource.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 04:16 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('aws', '0002_alter_awsresource_aws_region'), - ('case', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='AWSLogSource', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('service_name', models.CharField(max_length=100)), - ('log_name', models.CharField(max_length=255)), - ('log_details', models.JSONField(blank=True, null=True)), - ('status', models.CharField(max_length=50)), - ('aws_region', models.CharField(blank=True, max_length=50, null=True)), - ('slug', models.SlugField(blank=True, max_length=255, unique=True)), - ('discovered_at', models.DateTimeField(auto_now_add=True)), - ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='log_sources', to='aws.awsaccount')), - ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aws_log_sources', to='case.case')), - ], - ), - ] diff --git a/apps/aws/models.py b/apps/aws/models.py index 4ce1a3b..d422428 100644 --- a/apps/aws/models.py +++ b/apps/aws/models.py @@ -2,6 +2,7 @@ from django.conf import settings from apps.case.models import Case from django.utils.text import slugify +from apps.data.models import Tag # Used to show and access the AWS account for the case class AWSAccount(models.Model): @@ -29,6 +30,9 @@ class AWSResource(models.Model): slug = models.SlugField(max_length=255, unique=True, blank=True) discovered_at = models.DateTimeField(auto_now_add=True) + # Tags + tags = models.ManyToManyField(Tag, related_name='aws_resource') + def save(self, *args, **kwargs): if not self.slug: base_slug = slugify(f"{self.resource_type}-{self.resource_id}") @@ -55,6 +59,9 @@ class AWSLogSource(models.Model): slug = models.SlugField(max_length=255, unique=True, blank=True) discovered_at = models.DateTimeField(auto_now_add=True) + # Tags + tags = models.ManyToManyField(Tag, related_name='aws_log_source') + def save(self, *args, **kwargs): if not self.slug: base_slug = slugify(f"{self.service_name}-{self.log_name}") @@ -68,3 +75,56 @@ def save(self, *args, **kwargs): def __str__(self): return f"{self.service_name} - {self.log_name or self.status} for Account {self.account.account_id}" + +# Model to store credentials pulled from credential report api +class AWSCredential(models.Model): + account = models.ForeignKey(AWSAccount, on_delete=models.CASCADE, related_name='credentials') + case = models.ForeignKey(Case, on_delete=models.CASCADE, related_name='aws_credentials') + user = models.CharField(max_length=300) + user_arn = models.CharField(max_length=300) + user_creation_time = models.DateTimeField(null=True, blank=True) + password_enabled = models.BooleanField(default=False) + password_last_used = models.DateTimeField(null=True, blank=True) + password_last_changed = models.DateTimeField(null=True, blank=True) + password_next_rotation_date = models.DateTimeField(null=True, blank=True) + mfa_active = models.BooleanField(default=False) + access_key_1_active = models.BooleanField(default=False) + access_key_1_last_rotated = models.DateTimeField(null=True, blank=True) + access_key_1_last_used_date = models.DateTimeField(null=True, blank=True) + access_key_1_last_used_region = models.CharField(max_length=300, null=True, blank=True) + access_key_1_last_used_service = models.CharField(max_length=300, null=True, blank=True) + access_key_2_active = models.BooleanField(default=False) + access_key_2_last_rotated = models.DateTimeField(null=True, blank=True) + access_key_2_last_used_date = models.DateTimeField(null=True, blank=True) + access_key_2_last_used_region = models.CharField(max_length=300, null=True, blank=True) + access_key_2_last_used_service = models.CharField(max_length=300, null=True, blank=True) + cert_1_active = models.BooleanField(default=False) + cert_1_last_rotated = models.DateTimeField(null=True, blank=True) + cert_2_active = models.BooleanField(default=False) + cert_2_last_rotated = models.DateTimeField(null=True, blank=True) + slug = models.SlugField(max_length=255, unique=True, blank=True) + discovered_at = models.DateTimeField(auto_now_add=True) + + # Tags + tags = models.ManyToManyField(Tag, related_name='aws_credential') + + class Meta: + unique_together = ('account', 'user') + + def save(self, *args, **kwargs): + if not self.slug: + base_slug = slugify(f"{self.user}-{self.user_arn}") + unique_slug = base_slug + num = 1 + while AWSCredential.objects.filter(slug=unique_slug).exists(): + unique_slug = f"{base_slug}-{num}" + num += 1 + self.slug = unique_slug + super().save(*args, **kwargs) + + def __str__(self): + return f"{self.user} - {self.user_arn} for Account {self.account.account_id}" + + + + diff --git a/apps/aws/urls.py b/apps/aws/urls.py index 521336f..656d11b 100644 --- a/apps/aws/urls.py +++ b/apps/aws/urls.py @@ -11,11 +11,12 @@ path('accounts//delete/', views.delete_account, name='delete_account'), path('accounts//pull-resources/', views.pull_resources_view, name='pull_aws_resources'), path('resources//details/', views.aws_resource_details, name='aws_resource_details'), - path('accounts//account-details/', views.account_details, name='account_details'), + path('accounts//account-resources/', views.account_resources, name='account_resources'), path('logsource///details/', views.aws_logsource_details, name='aws_logsource_details'), path('fetch-management-events//', views.trigger_management_event_fetch, name='fetch_management_events'), path('browse-s3-structure/', views.browse_s3_structure, name='browse_s3_structure'), path('fetch-logs//', views.fetch_cloudtrail_logs, name='fetch_cloudtrail_logs'), - + path('accounts//logs/', views.normalized_logs_view, name='normalized_logs'), + path('credential//', views.aws_credential_details, name='aws_credential_details'), ] diff --git a/apps/aws/utils.py b/apps/aws/utils.py index d782518..839f789 100644 --- a/apps/aws/utils.py +++ b/apps/aws/utils.py @@ -1,18 +1,32 @@ import boto3 -from .models import AWSResource, AWSLogSource, AWSAccount +from .models import AWSResource, AWSLogSource, AWSAccount, AWSCredential from apps.data.models import NormalizedLog from apps.case.models import Case from datetime import datetime, timedelta -import datetime, gzip +from django.utils import timezone +import gzip from botocore.exceptions import EndpointConnectionError, ClientError import logging from django.db import transaction -from django.utils import timezone +from datetime import timezone +from django.utils.timezone import make_aware +from datetime import datetime, timezone +import time import json - +import ipaddress logger = logging.getLogger(__name__) +def parse_aws_datetime(datetime_str): + if datetime_str and datetime_str not in ['N/A', 'not_supported']: + try: + dt = datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%SZ') + return make_aware(dt, timezone.utc) + except ValueError: + logger.debug(f"Could not parse datetime: {datetime_str}") + return None + return None + # Validate AWS credentials by calling the STS GetCallerIdentity API. def validate_aws_credentials(aws_access_key, aws_secret_key, region): @@ -46,6 +60,82 @@ def pull_aws_resources(aws_account): aws_secret_access_key=aws_account.aws_secret_key ) + # Add new function to fetch IAM credential report + def fetch_credential_report(): + try: + iam = session.client('iam') + + # Generate credential report - this may take a few seconds + response = iam.generate_credential_report() + + # Wait for report to be generated + state = response.get('State', '') + while state != 'COMPLETE': + logger.info(f"Waiting for credential report to be generated... Current state: {state}") + time.sleep(2) + try: + response = iam.get_credential_report() + state = response.get('State', '') + except iam.exceptions.CredentialReportNotPresentException: + logger.info("Report not ready yet, retrying...") + continue + + # Get the credential report + response = iam.get_credential_report() + if 'Content' not in response: + logger.error("No content in credential report response") + return + + report_csv = response['Content'].decode('utf-8') + + # Parse CSV content + report_lines = report_csv.split('\n') + headers = report_lines[0].split(',') + + # Process each user's credentials + for line in report_lines[1:]: + if not line: + continue + + user_data = dict(zip(headers, line.split(','))) + + # Create or update AWSCredential record + AWSCredential.objects.update_or_create( + account=aws_account, + case=aws_account.case, + user=user_data['user'], + defaults={ + 'user_arn': user_data.get('arn', ''), + 'user_creation_time': parse_aws_datetime(user_data.get('user_creation_time')), + 'password_enabled': user_data.get('password_enabled', 'false').lower() == 'true', + 'password_last_used': parse_aws_datetime(user_data.get('password_last_used')), + 'password_last_changed': parse_aws_datetime(user_data.get('password_last_changed')), + 'password_next_rotation_date': parse_aws_datetime(user_data.get('password_next_rotation')), + 'mfa_active': user_data.get('mfa_active', 'false').lower() == 'true', + 'access_key_1_active': user_data.get('access_key_1_active', 'false').lower() == 'true', + 'access_key_1_last_rotated': parse_aws_datetime(user_data.get('access_key_1_last_rotated')), + 'access_key_1_last_used_date': parse_aws_datetime(user_data.get('access_key_1_last_used_date')), + 'access_key_1_last_used_region': user_data.get('access_key_1_last_used_region', ''), + 'access_key_1_last_used_service': user_data.get('access_key_1_last_used_service', ''), + 'access_key_2_active': user_data.get('access_key_2_active', 'false').lower() == 'true', + 'access_key_2_last_rotated': parse_aws_datetime(user_data.get('access_key_2_last_rotated')), + 'access_key_2_last_used_date': parse_aws_datetime(user_data.get('access_key_2_last_used_date')), + 'access_key_2_last_used_region': user_data.get('access_key_2_last_used_region', ''), + 'access_key_2_last_used_service': user_data.get('access_key_2_last_used_service', ''), + 'cert_1_active': user_data.get('cert_1_active', 'false').lower() == 'true', + 'cert_1_last_rotated': parse_aws_datetime(user_data.get('cert_1_last_rotated')), + 'cert_2_active': user_data.get('cert_2_active', 'false').lower() == 'true', + 'cert_2_last_rotated': parse_aws_datetime(user_data.get('cert_2_last_rotated')), + } + ) + logger.info("Successfully processed credential report") + + except Exception as e: + logger.error(f"Error fetching credential report: {e}") + + # Try to fetch credential report + fetch_credential_report() + # Fetch resources by looping through dynamically fetched regions def fetch_resources_by_region(service_name, fetch_function): try: @@ -336,103 +426,154 @@ def fetch_guardduty_detectors(): logger.info(f"Completed discovering log sources for AWS account: {aws_account.account_id}") -# this function extracts logs from a s3 bucket based on the users selected prefix +def normalize_cloudtrail_event(raw_event, case, aws_account): + """Helper function to normalize CloudTrail events into a consistent format""" + # Handle Records array if present + if 'Records' in raw_event: + raw_event = raw_event['Records'][0] # Take the first record + + # If this is from LookupEvents API, the actual event is in CloudTrailEvent + if 'CloudTrailEvent' in raw_event: + try: + raw_event = json.loads(raw_event['CloudTrailEvent']) + except (json.JSONDecodeError, TypeError): + pass + + # Extract event time + event_time = raw_event.get('eventTime') + if event_time: + event_time = parse_aws_datetime(event_time) + + # Extract user identity - just get the userName + user_identity = raw_event.get('userIdentity', {}) + username = ( + user_identity.get('userName') or + user_identity.get('sessionContext', {}).get('sessionIssuer', {}).get('userName') or + user_identity.get('invokedBy') or + user_identity.get('type') or + 'Unknown' + ) + + # Get resources directly from CloudTrail event + resources = raw_event.get('resources', []) + + # If no resources field, try to extract from request/response + if not resources: + request_params = raw_event.get('requestParameters', {}) + response_elements = raw_event.get('responseElements', {}) + + # Store as raw data to preserve all information + if request_params or response_elements: + resources = [{ + 'requestParameters': request_params, + 'responseElements': response_elements + }] + + # Validate and process the source IP address + source_ip = raw_event.get('sourceIPAddress') + if source_ip: + try: + # This will raise a ValueError if the IP is not valid + ipaddress.ip_address(source_ip) + except ValueError: + logger.debug(f"Invalid source IP: {source_ip}") + source_ip = None # Or handle it as needed + + # Build normalized log data + normalized_data = { + 'case': case, + 'aws_account': aws_account, + 'file_name': raw_event.get('s3', {}).get('object', {}).get('key'), + 'event_id': raw_event.get('eventID'), + 'event_time': event_time, + 'event_source': raw_event.get('eventSource'), + 'event_name': raw_event.get('eventName'), + 'event_type': raw_event.get('eventType'), + 'user_identity': username, + 'region': raw_event.get('awsRegion'), + 'ip_address': source_ip, # Now validated + 'user_agent': raw_event.get('userAgent'), + 'resources': json.dumps(resources), + 'raw_data': json.dumps(raw_event) + } + + return normalized_data + def fetch_and_normalize_cloudtrail_logs(account_id, resource_id, prefix, start_date, end_date, case_id): + """Fetch and normalize CloudTrail logs from S3""" try: - account = AWSAccount.objects.get(account_id=account_id) + aws_account = AWSAccount.objects.get(account_id=account_id) + case = Case.objects.get(id=case_id) resource = AWSResource.objects.get(id=resource_id) - except (AWSAccount.DoesNotExist, AWSResource.DoesNotExist): - print("DEBUG: Account or Resource not found.") + except (AWSAccount.DoesNotExist, AWSResource.DoesNotExist, Case.DoesNotExist): + logger.error("Account, Resource, or Case not found.") return session = boto3.Session( - aws_access_key_id=account.aws_access_key, - aws_secret_access_key=account.aws_secret_key, - region_name=resource.aws_region or account.default_region + aws_access_key_id=aws_account.aws_access_key, + aws_secret_access_key=aws_account.aws_secret_key, + region_name=resource.aws_region or aws_account.aws_region ) s3 = session.client("s3") bucket_name = resource.resource_name or resource.resource_id - # Convert date strings to date objects - start_date_obj = datetime.datetime.strptime(start_date, "%Y-%m-%d").date() - end_date_obj = datetime.datetime.strptime(end_date, "%Y-%m-%d").date() + start_date_obj = datetime.strptime(start_date, "%Y-%m-%d").date() + end_date_obj = datetime.strptime(end_date, "%Y-%m-%d").date() - # Ensure prefix ends with '/' if prefix and not prefix.endswith("/"): prefix += "/" current_date = start_date_obj - while current_date <= end_date_obj: - date_folder = f"{current_date.year}/{current_date.strftime('%m')}/{current_date.strftime('%d')}/" - final_prefix = prefix + date_folder - - print(f"DEBUG: Checking prefix '{final_prefix}' in bucket '{bucket_name}'") - - paginator = s3.get_paginator("list_objects_v2") - page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=final_prefix) - - any_objects_found = False - for page in page_iterator: - contents = page.get("Contents", []) - if not contents: - print(f"DEBUG: No objects in prefix '{final_prefix}' on this page.") - continue - - any_objects_found = True - - for obj in contents: - key = obj["Key"] - try: - resp = s3.get_object(Bucket=bucket_name, Key=key) - except Exception as e: - print(f"DEBUG: Error fetching object '{key}' - {e}") - continue - - raw_body = resp["Body"].read() - # Decompress if .gz - if key.endswith(".gz"): - try: - file_data = gzip.decompress(raw_body).decode("utf-8") - except OSError: - print(f"DEBUG: Failed to decompress '{key}'. Skipping.") - continue - else: - file_data = raw_body.decode("utf-8") - - # Parse CloudTrail JSON - try: - records_json = json.loads(file_data) - records = records_json.get("Records", []) - except json.JSONDecodeError: - print(f"DEBUG: JSON decode error in '{key}'. Skipping.") - continue - - print(f"DEBUG: Found {len(records)} log records in file '{key}'") - for record in records: - NormalizedLog.objects.create( - case_id=case_id, - log_id=record.get("eventID", ""), - log_source="aws", - log_type="CloudTrail", - event_name=record.get("eventName", ""), - event_time=record.get("eventTime", timezone.now()), - user_identity=record.get("userIdentity", {}), - ip_address=record.get("sourceIPAddress"), - resources=record.get("resources", []), - raw_data=record, - ) - - if not any_objects_found: - print(f"DEBUG: No objects found at all for prefix '{final_prefix}'") + with transaction.atomic(): + while current_date <= end_date_obj: + date_folder = f"{current_date.year}/{current_date.strftime('%m')}/{current_date.strftime('%d')}/" + final_prefix = prefix + date_folder - current_date += datetime.timedelta(days=1) + logger.info(f"Checking prefix '{final_prefix}' in bucket '{bucket_name}'") - print("DEBUG: Finished fetching logs.") + try: + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket_name, Prefix=final_prefix): + for obj in page.get("Contents", []): + key = obj["Key"] + try: + resp = s3.get_object(Bucket=bucket_name, Key=key) + raw_body = resp["Body"].read() + + # Decompress if gzipped + if key.endswith(".gz"): + file_data = gzip.decompress(raw_body).decode("utf-8") + else: + file_data = raw_body.decode("utf-8") + + records = json.loads(file_data).get("Records", []) + + # Process each record + for record in records: + try: + normalized_data = normalize_cloudtrail_event(record, case, aws_account) + NormalizedLog.objects.create(**normalized_data) + except Exception as e: + logger.error(f"Error processing record: {e}") + continue + + except Exception as e: + logger.error(f"Error processing file {key}: {e}") + continue + + except Exception as e: + logger.error(f"Error processing date {current_date}: {e}") + + current_date += timedelta(days=1) def fetch_management_event_history(account_id, case_id): - - aws_account = AWSAccount.objects.get(account_id=account_id) - case = Case.objects.get(id=case_id) + """Fetch and normalize CloudTrail management events using LookupEvents API""" + try: + aws_account = AWSAccount.objects.get(account_id=account_id) + case = Case.objects.get(id=case_id) + except (AWSAccount.DoesNotExist, Case.DoesNotExist) as e: + logger.error(f"Error fetching account or case: {e}") + return session = boto3.Session( aws_access_key_id=aws_account.aws_access_key, @@ -440,37 +581,38 @@ def fetch_management_event_history(account_id, case_id): region_name=aws_account.aws_region ) client = session.client('cloudtrail') - - # Paginator for management event history paginator = client.get_paginator('lookup_events') - with transaction.atomic(): + # Create a list to store normalized events before bulk create + normalized_events = [] + batch_size = 1000 # Process 1000 events at a time + + try: for page in paginator.paginate(): for event in page.get('Events', []): - # Ensure event data is JSON serializable - raw_event = json.loads(json.dumps(event, default=str)) - - log_data = { - 'case_id': case_id, - 'log_id': raw_event.get('EventId'), - 'log_source': 'aws', - 'log_type': raw_event.get('EventSource'), - 'event_name': raw_event.get('EventName'), - 'event_time': raw_event.get('EventTime'), - 'user_identity': raw_event.get('Username', {}), - 'ip_address': raw_event.get('SourceIPAddress'), - 'resources': raw_event.get('Resources', []), - 'raw_data': raw_event, # Ensure all fields are JSON serializable - 'extra_data': {}, # Add additional metadata if needed - } + try: + normalized_data = normalize_cloudtrail_event(event, case, aws_account) + # Create NormalizedLog instance but don't save yet + normalized_log = NormalizedLog(**normalized_data) + normalized_events.append(normalized_log) + + # When we reach batch_size, bulk create the records + if len(normalized_events) >= batch_size: + with transaction.atomic(): + NormalizedLog.objects.bulk_create(normalized_events) + normalized_events = [] # Clear the list after bulk create + + except Exception as e: + logger.error(f"Error processing event: {e}") + continue + + # Bulk create any remaining events + if normalized_events: + with transaction.atomic(): + NormalizedLog.objects.bulk_create(normalized_events) - # Normalize and save the log - normalized_log = NormalizedLog.objects.create(**log_data) + except Exception as e: + logger.error(f"Error fetching management events: {e}") - # Link to resources if applicable - for resource in raw_event.get('Resources', []): - aws_resource = AWSResource.objects.filter(resource_name=resource.get('ResourceName')).first() - if aws_resource: - normalized_log.aws_resources.add(aws_resource) diff --git a/apps/aws/views.py b/apps/aws/views.py index 664f52b..9c93512 100644 --- a/apps/aws/views.py +++ b/apps/aws/views.py @@ -1,16 +1,20 @@ import boto3 from django.shortcuts import render, redirect, get_object_or_404 from django.contrib.auth.decorators import login_required -from .models import AWSAccount, AWSResource, AWSLogSource +from .models import AWSAccount, AWSResource, AWSLogSource, AWSCredential from .forms import AWSAccountForm, FetchCloudTrailLogsForm from apps.case.models import Case from .utils import validate_aws_credentials from django.contrib import messages from .tasks import pull_aws_resources_task, fetch_management_history_task, fetch_normalize_cloudtrail_logs_task -from datetime import datetime +from datetime import datetime, timedelta from django.utils import timezone from django.http import JsonResponse from django.utils.timezone import make_aware +from apps.data.models import NormalizedLog +from django.db.models import Count, Value +from django.db.models.functions import Coalesce + import logging @@ -45,7 +49,7 @@ def connect_aws(request, slug): messages.error(request, f"AWS account saved, but validation failed: {error_message}") # Redirect to connected accounts page - return redirect('case:list_connected_accounts', slug=case.slug) + return redirect('case:case_detail', slug=case.slug) else: form = AWSAccountForm() @@ -59,7 +63,7 @@ def edit_account(request, account_id): form = AWSAccountForm(request.POST, instance=account) if form.is_valid(): form.save() - return redirect('case:list_connected_accounts', slug=account.case.slug) + return redirect('case:case_detail', slug=account.case.slug) else: form = AWSAccountForm(instance=account) @@ -71,7 +75,7 @@ def delete_account(request, account_id): account = get_object_or_404(AWSAccount, id=account_id) slug = account.case.slug # Save the slug for redirection account.delete() - return redirect('case:list_connected_accounts', slug=slug) + return redirect('case:case_detail', slug=slug) #This is the trigger for getting the resources of the AWS account. @@ -82,13 +86,13 @@ def pull_resources_view(request, account_id): if not aws_account.validated: messages.error(request, "Cannot pull resources because the AWS account credentials are not validated.") - return redirect('case:list_connected_accounts', slug=aws_account.case.slug) + return redirect('case:case_detail', slug=aws_account.case.slug) # Trigger background task pull_aws_resources_task.delay(account_id) - messages.info(request, "Resource pulling has started. This may take a few minutes.") + messages.info(request, "Resource pulling has started. Refresh the page after after a few minutes to see the results.") - return redirect('case:list_connected_accounts', slug=aws_account.case.slug) + return redirect('aws:account_resources', account_id=aws_account.account_id) # Open a modal to show the details of the resources @login_required @@ -101,8 +105,9 @@ def aws_resource_details(request, resource_id): # this renders both the aws resources and logging sources into one page @login_required -def account_details(request, account_id): - aws_account = get_object_or_404(AWSAccount, id=account_id) +def account_resources(request, account_id): + aws_account = get_object_or_404(AWSAccount, account_id=account_id) + case = aws_account.case # Group resources by their type resources = AWSResource.objects.filter(account=aws_account).order_by('resource_type', 'resource_name') @@ -123,13 +128,58 @@ def account_details(request, account_id): if not log_sources.exists(): error_messages.append("No AWS log sources found for this account.") + aws_credentials = AWSCredential.objects.filter(account=aws_account) + context = { 'aws_account': aws_account, + 'case': case, 'grouped_resources': grouped_resources, 'grouped_log_sources': grouped_log_sources, 'error_messages': error_messages, + 'aws_credentials': aws_credentials, } - return render(request, 'aws/account_details.html', context) + return render(request, 'aws/account_resources.html', context) + + +@login_required +def normalized_logs_view(request, account_id): + aws_account = get_object_or_404(AWSAccount, account_id=account_id) + + # Default date filter: Last day + end_date = datetime.now() + start_date = request.GET.get("start_date", (end_date - timedelta(days=1)).strftime("%Y-%m-%d")) + end_date = request.GET.get("end_date", end_date.strftime("%Y-%m-%d")) + + # Convert to date objects + start_date = datetime.strptime(start_date, "%Y-%m-%d").date() + end_date = datetime.strptime(end_date, "%Y-%m-%d").date() + + # Filter logs for the specific AWSAccount within the date range + logs = NormalizedLog.objects.filter( + aws_account=aws_account, + event_time__date__gte=start_date, + event_time__date__lte=end_date + ) + + # Aggregate top 10 users + top_users = logs.values('user_identity').annotate(count=Count('user_identity')).order_by('-count')[:10] + + # Aggregate top 10 IPs + top_ips = logs.values('ip_address').annotate(count=Count('ip_address')).order_by('-count')[:10] + + # Aggregate top 10 events + top_events = logs.values('event_name').annotate(count=Count('event_name')).order_by('-count')[:10] + + context = { + "aws_account": aws_account, + "logs": logs, + "top_users": top_users, + "top_ips": top_ips, + "top_events": top_events, + "start_date": start_date, + "end_date": end_date, + } + return render(request, "aws/normalized_logs.html", context) @login_required def aws_logsource_details(request, slug): @@ -183,7 +233,7 @@ def fetch_cloudtrail_logs(request, account_id): if resource.account.account_id != aws_account.account_id: messages.error(request, "Selected bucket is not linked to this AWS account.") - return redirect("case:list_connected_accounts", slug=resource.case.slug) + return redirect("case:case_detail", slug=resource.case.slug) fetch_normalize_cloudtrail_logs_task.delay( account_id=aws_account.account_id, @@ -194,7 +244,7 @@ def fetch_cloudtrail_logs(request, account_id): case_id=resource.case.id ) messages.success(request, "CloudTrail log fetching has been queued.") - return redirect("case:list_connected_accounts", slug=resource.case.slug) + return redirect("aws:normalized_logs", account_id=aws_account.account_id) else: form = FetchCloudTrailLogsForm() @@ -210,5 +260,20 @@ def trigger_management_event_fetch(request, account_id): messages.success(request, "Management event history is being fetched.") logger.info(f"Task queued for AWS account {account_id}") - return redirect('case:list_connected_accounts', slug=aws_account.case.slug) + return redirect("aws:normalized_logs", account_id=aws_account.account_id) + +@login_required +def aws_credential_details(request, slug): + """ + Display detailed information for a specific IAM credential. + """ + credential = get_object_or_404(AWSCredential, slug=slug) + + context = { + 'credential': credential, + 'case': credential.case, + 'aws_account': credential.account, + } + + return render(request, 'aws/credential_details.html', context) diff --git a/apps/case/migrations/0001_initial.py b/apps/case/migrations/0001_initial.py index f748a05..a099624 100644 --- a/apps/case/migrations/0001_initial.py +++ b/apps/case/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.3 on 2024-12-25 11:56 +# Generated by Django 5.1.3 on 2025-01-22 05:31 import django.db.models.deletion import uuid diff --git a/apps/case/urls.py b/apps/case/urls.py index 7b881d8..92fbcfa 100644 --- a/apps/case/urls.py +++ b/apps/case/urls.py @@ -10,6 +10,5 @@ path('/', views.case_detail, name='case_detail'), path('/edit/', views.edit_case, name='edit_case'), path('/connect/', views.connect_client, name='connect_client'), - path('/connected-accounts/', views.list_connected_accounts, name='list_connected_accounts'), ] diff --git a/apps/case/views.py b/apps/case/views.py index 125ae78..a4c58bc 100644 --- a/apps/case/views.py +++ b/apps/case/views.py @@ -23,8 +23,21 @@ def create_case(request): # This is used to view the details of a case @login_required def case_detail(request, slug): - case = Case.objects.get(slug=slug) - return render(request, "case/case_detail.html", {"case": case}) + case = get_object_or_404(Case, slug=slug) + + # AWS accounts linked to the case + aws_accounts = AWSAccount.objects.filter(case=case) + + # Add GCP and Azure placeholders + gcp_placeholder = True + azure_placeholder = True + + return render(request, "case/case_detail.html", { + "case": case, + "aws_accounts": aws_accounts, + "gcp_placeholder": gcp_placeholder, + "azure_placeholder": azure_placeholder, + }) # this is used to edit the details of a case @login_required @@ -50,20 +63,3 @@ def edit_case(request, slug): def connect_client(request, slug): case = get_object_or_404(Case, slug=slug) return render(request, 'case/connect_client.html', {'case': case}) - -@login_required -def list_connected_accounts(request, slug): - case = get_object_or_404(Case, slug=slug) - - # AWS accounts linked to the case - aws_accounts = AWSAccount.objects.filter(case=case) - - # Add GCP and azure later - - return render(request, 'case/list_connected_accounts.html', { - 'case': case, - 'aws_accounts': aws_accounts, - # GCP and Azure placeholders - 'gcp_placeholder': True, - 'azure_placeholder': True, - }) \ No newline at end of file diff --git a/apps/data/__init__.py b/apps/data/__init__.py index e69de29..0827d7e 100644 --- a/apps/data/__init__.py +++ b/apps/data/__init__.py @@ -0,0 +1 @@ +default_app_config = 'apps.data.apps.DataConfig' diff --git a/apps/data/admin.py b/apps/data/admin.py index 72541b1..8e47d97 100644 --- a/apps/data/admin.py +++ b/apps/data/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from .models import NormalizedLog +from .models import NormalizedLog, Tag # Register your models here. -admin.site.register(NormalizedLog) \ No newline at end of file +admin.site.register(NormalizedLog) +admin.site.register(Tag) \ No newline at end of file diff --git a/apps/data/management/__init__.py b/apps/data/management/__init__.py new file mode 100644 index 0000000..1c705ce --- /dev/null +++ b/apps/data/management/__init__.py @@ -0,0 +1 @@ +# Empty file \ No newline at end of file diff --git a/apps/data/management/commands/__init__.py b/apps/data/management/commands/__init__.py new file mode 100644 index 0000000..1c705ce --- /dev/null +++ b/apps/data/management/commands/__init__.py @@ -0,0 +1 @@ +# Empty file \ No newline at end of file diff --git a/apps/data/management/commands/init_tags.py b/apps/data/management/commands/init_tags.py new file mode 100644 index 0000000..47a848f --- /dev/null +++ b/apps/data/management/commands/init_tags.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand +from apps.data.models import Tag + +class Command(BaseCommand): + help = 'Initialize default tags' + + def handle(self, *args, **kwargs): + self.stdout.write('Creating default tags...') + Tag.init_tags() + self.stdout.write(self.style.SUCCESS('Successfully created default tags')) \ No newline at end of file diff --git a/apps/data/migrations/0001_initial.py b/apps/data/migrations/0001_initial.py index 6eb1dc2..38b02ee 100644 --- a/apps/data/migrations/0001_initial.py +++ b/apps/data/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 5.1.3 on 2024-12-25 11:56 +# Generated by Django 5.1.3 on 2025-01-22 05:31 +import django.contrib.postgres.indexes import django.db.models.deletion from django.db import migrations, models @@ -18,17 +19,22 @@ class Migration(migrations.Migration): name='NormalizedLog', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('log_source', models.CharField(choices=[('aws', 'Amazon Web Services'), ('gcp', 'Google Cloud Platform'), ('azure', 'Microsoft Azure')], max_length=50)), - ('log_type', models.CharField(max_length=100)), - ('event_name', models.CharField(max_length=255)), - ('event_time', models.DateTimeField()), - ('user_identity', models.JSONField(blank=True, null=True)), - ('resources', models.JSONField(blank=True, null=True)), - ('raw_data', models.JSONField()), - ('extra_data', models.JSONField(blank=True, null=True)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('aws_resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='normalized_logs', to='aws.awsresource')), + ('log_id', models.CharField(blank=True, max_length=255, null=True)), + ('log_source', models.CharField(choices=[('aws', 'Amazon Web Services'), ('gcp', 'Google Cloud Platform'), ('azure', 'Microsoft Azure')], db_index=True, max_length=50)), + ('log_type', models.CharField(db_index=True, max_length=100)), + ('event_name', models.CharField(db_index=True, max_length=255)), + ('event_time', models.DateTimeField(db_index=True)), + ('user_identity', models.CharField(blank=True, max_length=255, null=True)), + ('ip_address', models.GenericIPAddressField(blank=True, db_index=True, null=True)), + ('resources', models.TextField(blank=True, null=True)), + ('raw_data', models.TextField()), + ('extra_data', models.TextField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('aws_account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='normalized_logs', to='aws.awsaccount')), ('case', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='normalized_logs', to='case.case')), ], + options={ + 'indexes': [django.contrib.postgres.indexes.BTreeIndex(fields=['log_source'], name='data_normal_log_sou_bee0df_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['log_type'], name='data_normal_log_typ_a9e795_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['event_name'], name='data_normal_event_n_8b8bea_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['event_time'], name='data_normal_event_t_c05244_btree'), django.contrib.postgres.indexes.BTreeIndex(fields=['ip_address'], name='data_normal_ip_addr_3df897_btree')], + }, ), ] diff --git a/apps/data/migrations/0002_remove_normalizedlog_aws_resource_and_more.py b/apps/data/migrations/0002_remove_normalizedlog_aws_resource_and_more.py deleted file mode 100644 index 61546fd..0000000 --- a/apps/data/migrations/0002_remove_normalizedlog_aws_resource_and_more.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 02:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('aws', '0002_alter_awsresource_aws_region'), - ('data', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='normalizedlog', - name='aws_resource', - ), - migrations.AddField( - model_name='normalizedlog', - name='aws_resources', - field=models.ManyToManyField(related_name='normalized_logs', to='aws.awsresource'), - ), - migrations.AddField( - model_name='normalizedlog', - name='ip_address', - field=models.GenericIPAddressField(blank=True, null=True), - ), - ] diff --git a/apps/data/migrations/0002_tag_and_more.py b/apps/data/migrations/0002_tag_and_more.py new file mode 100644 index 0000000..d1cb788 --- /dev/null +++ b/apps/data/migrations/0002_tag_and_more.py @@ -0,0 +1,106 @@ +# Generated by Django 5.1.3 on 2025-01-24 03:03 + +import django.contrib.postgres.indexes +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('aws', '0001_initial'), + ('case', '0001_initial'), + ('data', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Tag', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=1000)), + ('description', models.TextField(blank=True, null=True)), + ('slug', models.SlugField(max_length=1000, unique=True)), + ], + ), + migrations.RemoveIndex( + model_name='normalizedlog', + name='data_normal_log_sou_bee0df_btree', + ), + migrations.RemoveIndex( + model_name='normalizedlog', + name='data_normal_log_typ_a9e795_btree', + ), + migrations.RemoveField( + model_name='normalizedlog', + name='extra_data', + ), + migrations.RemoveField( + model_name='normalizedlog', + name='log_id', + ), + migrations.RemoveField( + model_name='normalizedlog', + name='log_source', + ), + migrations.RemoveField( + model_name='normalizedlog', + name='log_type', + ), + migrations.AddField( + model_name='normalizedlog', + name='event_id', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AddField( + model_name='normalizedlog', + name='event_source', + field=models.CharField(blank=True, choices=[('aws', 'Amazon Web Services'), ('gcp', 'Google Cloud Platform'), ('azure', 'Microsoft Azure')], db_index=True, max_length=1000, null=True), + ), + migrations.AddField( + model_name='normalizedlog', + name='event_type', + field=models.CharField(blank=True, db_index=True, max_length=1000, null=True), + ), + migrations.AddField( + model_name='normalizedlog', + name='file_name', + field=models.CharField(blank=True, max_length=2000, null=True), + ), + migrations.AddField( + model_name='normalizedlog', + name='region', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AddField( + model_name='normalizedlog', + name='user_agent', + field=models.CharField(blank=True, max_length=3000, null=True), + ), + migrations.AlterField( + model_name='normalizedlog', + name='event_name', + field=models.CharField(blank=True, db_index=True, max_length=1000, null=True), + ), + migrations.AlterField( + model_name='normalizedlog', + name='event_time', + field=models.DateTimeField(blank=True, db_index=True, null=True), + ), + migrations.AlterField( + model_name='normalizedlog', + name='user_identity', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AddIndex( + model_name='normalizedlog', + index=django.contrib.postgres.indexes.BTreeIndex(fields=['event_source'], name='data_normal_event_s_9d3f1d_btree'), + ), + migrations.AddIndex( + model_name='normalizedlog', + index=django.contrib.postgres.indexes.BTreeIndex(fields=['user_agent'], name='data_normal_user_ag_bd607d_btree'), + ), + migrations.AddIndex( + model_name='normalizedlog', + index=django.contrib.postgres.indexes.BTreeIndex(fields=['region'], name='data_normal_region_72366d_btree'), + ), + ] diff --git a/apps/data/migrations/0003_normalizedlog_log_id.py b/apps/data/migrations/0003_normalizedlog_log_id.py deleted file mode 100644 index 99412be..0000000 --- a/apps/data/migrations/0003_normalizedlog_log_id.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.3 on 2025-01-08 02:30 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('data', '0002_remove_normalizedlog_aws_resource_and_more'), - ] - - operations = [ - migrations.AddField( - model_name='normalizedlog', - name='log_id', - field=models.CharField(blank=True, max_length=255, null=True), - ), - ] diff --git a/apps/data/migrations/0003_normalizedlog_tags.py b/apps/data/migrations/0003_normalizedlog_tags.py new file mode 100644 index 0000000..8fad1be --- /dev/null +++ b/apps/data/migrations/0003_normalizedlog_tags.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.3 on 2025-01-24 03:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0002_tag_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='normalizedlog', + name='tags', + field=models.ManyToManyField(related_name='normalized_logs', to='data.tag'), + ), + ] diff --git a/apps/data/migrations/0004_alter_normalizedlog_unique_together.py b/apps/data/migrations/0004_alter_normalizedlog_unique_together.py new file mode 100644 index 0000000..07fde5d --- /dev/null +++ b/apps/data/migrations/0004_alter_normalizedlog_unique_together.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.3 on 2025-01-24 04:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('case', '0001_initial'), + ('data', '0003_normalizedlog_tags'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='normalizedlog', + unique_together={('case', 'event_id')}, + ), + ] diff --git a/apps/data/models.py b/apps/data/models.py index da2dc50..c71a0e0 100644 --- a/apps/data/models.py +++ b/apps/data/models.py @@ -1,10 +1,30 @@ from django.db import models -from apps.aws.models import AWSResource -from apps.case.models import Case +from django.contrib.postgres.indexes import BTreeIndex + + +class Tag(models.Model): + name = models.CharField(max_length=1000) + description = models.TextField(blank=True, null=True) + slug = models.SlugField(max_length=1000, unique=True) + + def __str__(self): + return self.name + + @classmethod + def init_tags(cls): + tags = [ + {"name": "Suspicious", "slug": "suspicious"}, + {"name": "Malicious", "slug": "malicious"}, + {"name": "Informational", "slug": "informational"}, + {"name": "Follow-up", "slug": "follow-up"}, + {"name": "Low", "slug": "low"}, + {"name": "Medium", "slug": "medium"}, + {"name": "High", "slug": "high"}, + ] + for tag in tags: + cls.objects.get_or_create(name=tag["name"], slug=tag["slug"]) -# Create your models here. -# This stores all the logs in a normalised fashion. All logs from all services are normalized to allow for standard queries class NormalizedLog(models.Model): SOURCE = [ ("aws", "Amazon Web Services"), @@ -12,25 +32,42 @@ class NormalizedLog(models.Model): ("azure", "Microsoft Azure"), ] - case = models.ForeignKey(Case, on_delete=models.CASCADE, related_name='normalized_logs') - log_id = models.CharField(max_length=255, blank=True, null=True) # Log id from the service - log_source = models.CharField(max_length=50, choices=SOURCE) - log_type = models.CharField(max_length=100) - event_name = models.CharField(max_length=255) - event_time = models.DateTimeField() - user_identity = models.JSONField(blank=True, null=True) - ip_address = models.GenericIPAddressField(blank=True, null=True) - resources = models.JSONField(blank=True, null=True) - raw_data = models.JSONField() - extra_data = models.JSONField(blank=True, null=True) - - # Relationships - aws_resources = models.ManyToManyField(AWSResource, related_name='normalized_logs') + case = models.ForeignKey('case.Case', on_delete=models.CASCADE, related_name='normalized_logs') + file_name = models.CharField(max_length=2000, blank=True, null=True) + event_id = models.CharField(max_length=1000, blank=True, null=True) # Log ID from the service + event_time = models.DateTimeField(db_index=True, null=True, blank=True) # Indexed for date filtering + event_source = models.CharField(max_length=1000, choices=SOURCE, null=True, blank=True, db_index=True) # Indexed + event_name = models.CharField(max_length=1000, null=True, blank=True, db_index=True) # Indexed + event_type = models.CharField(max_length=1000, null=True, blank=True, db_index=True) # Indexed + user_identity = models.CharField(max_length=1000, blank=True, null=True) # Stores username or user info + region = models.CharField(max_length=1000, blank=True, null=True) # Indexed + ip_address = models.GenericIPAddressField(blank=True, null=True, db_index=True) # Indexed + user_agent = models.CharField(max_length=3000, blank=True, null=True) # Indexed + resources = models.TextField(blank=True, null=True) # Serialized list of resources as text + raw_data = models.TextField() # Serialized JSON as text + + # Use string reference to break circular import + aws_account = models.ForeignKey('aws.AWSAccount', on_delete=models.CASCADE, related_name='normalized_logs') + tags = models.ManyToManyField(Tag, related_name='normalized_logs') # Utility - created_at = models.DateTimeField(auto_now_add=True) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) # Indexed for sorting by creation time def __str__(self): - return f"{self.event_name} ({self.log_source}) - Case {self.case.name}" + return f"{self.event_name} ({self.event_source}) - Case {self.case.name}" + + + class Meta: + indexes = [ + BTreeIndex(fields=["event_source"]), + BTreeIndex(fields=["event_name"]), + BTreeIndex(fields=["event_time"]), + BTreeIndex(fields=["ip_address"]), + BTreeIndex(fields=["user_agent"]), + BTreeIndex(fields=["region"]), + ] + unique_together = (("case", "event_id"),) + + diff --git a/static/images/logo/icon.png b/static/images/logo/icon.png new file mode 100644 index 0000000..e5aa6dc Binary files /dev/null and b/static/images/logo/icon.png differ diff --git a/static/images/logo/icon.svg b/static/images/logo/icon.svg new file mode 100644 index 0000000..9256439 --- /dev/null +++ b/static/images/logo/icon.svg @@ -0,0 +1,9 @@ + + 2 + + + + + + \ No newline at end of file diff --git a/static/images/logo/logo.png b/static/images/logo/logo.png new file mode 100644 index 0000000..648a0ec Binary files /dev/null and b/static/images/logo/logo.png differ diff --git a/static/images/logo/logo.svg b/static/images/logo/logo.svg new file mode 100644 index 0000000..ccb9be3 --- /dev/null +++ b/static/images/logo/logo.svg @@ -0,0 +1,9 @@ + + 1 + + + + + + \ No newline at end of file diff --git a/templates/account/login.html b/templates/account/login.html index 5f56a9c..c71d620 100644 --- a/templates/account/login.html +++ b/templates/account/login.html @@ -12,8 +12,7 @@

{% translate "Sign In" %}

{% render_text_input form.login %} {% render_text_input form.password %} - - {% include 'account/components/social/social_buttons.html' %} + } {% if LOGIN_BY_CODE_ENABLED %}

diff --git a/templates/aws/account_details.html b/templates/aws/account_details.html deleted file mode 100644 index 11bdef5..0000000 --- a/templates/aws/account_details.html +++ /dev/null @@ -1,73 +0,0 @@ -{% extends "web/app/app_base.html" %} -{% load i18n %} -{% load static %} - -{% block app %} -

- -

Account Details for: {{ aws_account.account_id }}

- - -
-

AWS Resources

- - {% for resource_type, resources in grouped_resources.items %} -
-

{{ resource_type }}

- -
- {% for resource in resources %} -
-
-
-

- {{ resource.resource_name|default:resource.resource_id }} -

-

Region: {{ resource.aws_region }}

- - View Details - -
-
-
- {% endfor %} -
-
- {% endfor %} -
- - -
-

AWS Log Sources

- - {% for message in error_messages %} -
{{ message }}
- {% endfor %} - - {% for service_name, log_sources in grouped_log_sources.items %} -
-

{{ service_name }}

- -
- {% for log_source in log_sources %} -
-
-
-

{{ log_source.log_name }}

-

Region: {{ log_source.aws_region|default:"Global" }}

-

Status: {{ log_source.status }}

- - View Details - -
-
-
- {% endfor %} -
-
- {% endfor %} -
-
-{% endblock %} diff --git a/templates/aws/account_resources.html b/templates/aws/account_resources.html new file mode 100644 index 0000000..17645d4 --- /dev/null +++ b/templates/aws/account_resources.html @@ -0,0 +1,202 @@ +{% extends "web/app/app_base.html" %} +{% load i18n %} +{% load static %} + +{% block app %} +
+ + + + +
+

Account Details for: {{ aws_account.account_id }}

+ + Generate Overview + +
+ + +
+

AWS Resources

+ + {% if grouped_resources %} + {% for resource_type, resources in grouped_resources.items %} +
+

{{ resource_type }}

+ +
+ {% for resource in resources %} +
+
+
+

+ {{ resource.resource_name|default:resource.resource_id }} +

+

Region: {{ resource.aws_region }}

+ + View Details + +
+
+
+ {% endfor %} +
+
+ {% endfor %} + {% else %} +
+

No resources found. Would you like to pull resources?

+ + Generate Overview + +
+ {% endif %} +
+ + +
+

AWS Log Sources

+ + {% for message in error_messages %} +
{{ message }}
+ {% endfor %} + + {% if grouped_log_sources %} + {% for service_name, log_sources in grouped_log_sources.items %} +
+

{{ service_name }}

+ +
+ {% for log_source in log_sources %} +
+
+
+

{{ log_source.log_name }}

+

Region: {{ log_source.aws_region|default:"Global" }}

+

Status: {{ log_source.status }}

+ + View Details + +
+
+
+ {% endfor %} +
+
+ {% endfor %} + {% else %} +
+

No log sources found.

+
+ {% endif %} +
+ + + +
+
+
+ +
+
+
+
+
+ + + + + + + + + + + + + + {% for credential in aws_credentials %} + + + + + + + + + + {% empty %} + + + + {% endfor %} + +
UsernameCreatedPassword EnabledMFA ActiveAccess Key 1Access Key 2Last Activity
+ + {{ credential.user }} + + {{ credential.user_creation_time|default:"N/A" }} + {% if credential.password_enabled %} + Yes + {% else %} + No + {% endif %} + + {% if credential.mfa_active %} + Yes + {% else %} + No + {% endif %} + + {% if credential.access_key_1_active %} + Active + {% if credential.access_key_1_last_used_date %} +
Last used: {{ credential.access_key_1_last_used_date|date }} + {% endif %} + {% else %} + Inactive + {% endif %} + 		+ {% if credential.access_key_2_active %} + Active + {% if credential.access_key_2_last_used_date %} +
Last used: {{ credential.access_key_2_last_used_date|date }} + {% endif %} + {% else %} + Inactive + {% endif %} + 		+ {% if credential.password_last_used %} + Password: {{ credential.password_last_used|date }}
+ {% endif %} + {% if credential.access_key_1_last_used_date or credential.access_key_2_last_used_date %} + {% with last_key_use=credential.access_key_1_last_used_date|default:credential.access_key_2_last_used_date %} + Access Key: {{ last_key_use|date }} + {% endwith %} + {% endif %} +
No credentials found
+
+
+
+
+
+{% endblock %} diff --git a/templates/aws/credential_details.html b/templates/aws/credential_details.html new file mode 100644 index 0000000..04481b1 --- /dev/null +++ b/templates/aws/credential_details.html @@ -0,0 +1,162 @@ +{% extends "web/app/app_base.html" %} +{% load i18n %} +{% load static %} + +{% block app %} +
+ + + + +
+

IAM User Details: {{ credential.user }}

+
+ +
+ +
+
+
+

Basic Information

+
+
+
+
Username
+
{{ credential.user }}
+ +
ARN
+
{{ credential.user_arn }}
+ +
Created
+
{{ credential.user_creation_time|default:"N/A" }}
+
+
+
+
+ + +
+
+
+

Password Status

+
+
+
+
Password Enabled
+
+ {% if credential.password_enabled %} + Yes + {% else %} + No + {% endif %} +
+ +
Last Used
+
{{ credential.password_last_used|default:"Never" }}
+ +
Last Changed
+
{{ credential.password_last_changed|default:"Never" }}
+ +
Next Rotation
+
{{ credential.password_next_rotation_date|default:"Not Set" }}
+ +
MFA Active
+
+ {% if credential.mfa_active %} + Yes + {% else %} + No + {% endif %} +
+
+
+
+
+ + +
+
+
+

Access Key 1

+
+
+
+
Status
+
+ {% if credential.access_key_1_active %} + Active + {% else %} + Inactive + {% endif %} +
+ +
Last Rotated
+
{{ credential.access_key_1_last_rotated|default:"Never" }}
+ +
Last Used
+
{{ credential.access_key_1_last_used_date|default:"Never" }}
+ +
Last Used Region
+
{{ credential.access_key_1_last_used_region|default:"N/A" }}
+ +
Last Used Service
+
{{ credential.access_key_1_last_used_service|default:"N/A" }}
+
+
+
+
+ + +
+
+
+

Access Key 2

+
+
+
+
Status
+
+ {% if credential.access_key_2_active %} + Active + {% else %} + Inactive + {% endif %} +
+ +
Last Rotated
+
{{ credential.access_key_2_last_rotated|default:"Never" }}
+ +
Last Used
+
{{ credential.access_key_2_last_used_date|default:"Never" }}
+ +
Last Used Region
+
{{ credential.access_key_2_last_used_region|default:"N/A" }}
+ +
Last Used Service
+
{{ credential.access_key_2_last_used_service|default:"N/A" }}
+
+
+
+
+
+
+{% endblock %} \ No newline at end of file diff --git a/templates/aws/normalized_logs.html b/templates/aws/normalized_logs.html new file mode 100644 index 0000000..b175ecd --- /dev/null +++ b/templates/aws/normalized_logs.html @@ -0,0 +1,118 @@ +{% extends "web/app/app_base.html" %} +{% load i18n %} +{% load static %} + +{% block app %} +
+ + + + +
+

Logs for Account: {{ aws_account.account_id }}

+ +
+ + +
+
+
+ + +
+
+ + +
+
+ +
+
+
+ + +
+ +
+
+
+

Top 10 Users

+
    + {% for user in top_users %} +
  • + {{ user.user_identity|default:"Unknown User" }} + {{ user.count }} +
    • + {% empty %} +
  • + No users found. +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+

Top 10 IPs

+
    + {% for ip in top_ips %} +
  • + {{ ip.ip_address|default:"Unknown IP" }} + {{ ip.count }} +
    • + {% empty %} +
  • + No IPs found. +
    • + {% endfor %} +
+
+
+
+ + +
+
+
+

Top 10 Events

+
    + {% for event in top_events %} +
  • + {{ event.event_name|default:"Unknown Event" }} + {{ event.count }} +
    • + {% empty %} +
  • + No events found. +
    • + {% endfor %} +
+
+
+
+
+
+{% endblock %} diff --git a/templates/case/case_detail.html b/templates/case/case_detail.html index b1fea58..c5ecafa 100644 --- a/templates/case/case_detail.html +++ b/templates/case/case_detail.html @@ -3,24 +3,92 @@ {% load static %} {% block app %} -
-

{{ case.name }}

-

{{ case.description }}

-

Status: {{ case.status }}

-

Created by: {{ case.created_by.username }}

-

Created at: {{ case.created_at }}

- -
- Edit Case +
+ + + + +
+ +
+

{{ case.name }}

+

Status: {{ case.status }}

+

Created At: {{ case.created_at|date:"Y-m-d H:i" }}

+

Case ID: {{ case.uuid }}

+

{{ case.description }}

-
- Connect Client + + +
+ Edit Case + Connect Client
-
- - View Connections - +
+ + +

Connected Accounts

+ {% if aws_accounts %} +
+ {% for account in aws_accounts %} +
+
+ +
+
+ AWS Account: {{ account.account_id }} +
+

+ Region: {{ account.aws_region }} | + Added By: {{ account.added_by.username }} | + Added At: {{ account.added_at|date:"Y-m-d H:i" }} +

+ {% if account.validated %} + Validated + {% else %} + Not Validated + {% endif %} +
+ + +
+ Edit + + + Overview + + + Logging + +
+
-
+ {% endfor %} +
+ {% else %} +

No accounts connected yet. Use "Connect Client" to add accounts.

+ {% endif %} + + +

GCP Accounts

+ {% if gcp_placeholder %} +

GCP integration coming soon.

+ {% endif %} + +

Azure Accounts

+ {% if azure_placeholder %} +

Azure integration coming soon.

+ {% endif %} +
{% endblock %} diff --git a/templates/case/list_connected_accounts.html b/templates/case/list_connected_accounts.html deleted file mode 100644 index f737d93..0000000 --- a/templates/case/list_connected_accounts.html +++ /dev/null @@ -1,89 +0,0 @@ -{% extends "web/app/app_base.html" %} -{% load i18n %} -{% load static %} -{% block app %} - -
-

Connected Accounts for Case: {{ case.name }}

- - -
- Return to Case -
- -
- Connect Another Account -
- - -

AWS Accounts

- {% if aws_accounts %} -
- - - - - - - - - - - - {% for account in aws_accounts %} - - - - - - - - - {% endfor %} - -
Account IDRegionAdded ByAdded AtActions
- - {{ account.account_id }} - - {{ account.aws_region }}{{ account.added_by.username }}{{ account.added_at|date:"Y-m-d H:i" }} - {% if account.validated %} - Validated - {% else %} - Not Validated - {% endif %} - - Edit - - - Pull Resources - - - Get CloudTrail Logs - - - Get Management Logs (90 days) - -
-
- {% else %} -

No AWS accounts connected yet.

- {% endif %} - - -

GCP Accounts

- {% if gcp_placeholder %} -

GCP integration coming soon.

- {% endif %} - - -

Azure Accounts

- {% if azure_placeholder %} -

Azure integration coming soon.

- {% endif %} -
- -{% endblock %} diff --git a/templates/web/app/app_base.html b/templates/web/app/app_base.html index 4a55803..b795c27 100644 --- a/templates/web/app/app_base.html +++ b/templates/web/app/app_base.html @@ -3,14 +3,9 @@ {% block body %} {% block notifications %} {% endblock %} -
-
-
- {% block sidebar-nav %} - {% include "web/components/app_nav.html" %} - {% endblock %} -
-
+
+
+
{% block app %} {% endblock %}
diff --git a/templates/web/app_home.html b/templates/web/app_home.html index 1657af4..82314c0 100644 --- a/templates/web/app_home.html +++ b/templates/web/app_home.html @@ -4,39 +4,45 @@ {% block app %}
-
-

Your Cases

- -
- Create New Case +
+ +
+

Your Cases

+ Create New Case
+ {% if cases %} - - - - - - - - +
+
NameStatusCreated AtActions
+ + + + + + + - {% for case in cases %} - - - - - - - {% endfor %} + {% for case in cases %} + + + + + + + {% endfor %} -
NameStatusCreated AtActions
{{ case.name }}{{ case.status }}{{ case.created_at|date:"Y-m-d H:i" }} - View Details -
{{ case.name }}{{ case.status }}{{ case.created_at|date:"Y-m-d H:i" }} + View Case +
+ +
{% else %} -

You have no cases yet. Create a new one to get started.

+ {% endif %} -
+
+ {% endblock %} diff --git a/templates/web/components/footer.html b/templates/web/components/footer.html index 97cdef6..a6a240d 100644 --- a/templates/web/components/footer.html +++ b/templates/web/components/footer.html @@ -1,11 +1,9 @@ {% load i18n %}
-

{{project_meta.NAME}} — {% translate "Copyright" %} 2024

+

Scope Forensics

- + diff --git a/templates/web/components/top_nav.html b/templates/web/components/top_nav.html index 8b0811d..23a70b1 100644 --- a/templates/web/components/top_nav.html +++ b/templates/web/components/top_nav.html @@ -2,7 +2,7 @@