Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/topics/development/scanner_pipeline.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ The payload sent looks like this:
"download_source_url": "http://olympia.test/downloads/source/42",
"license_slug": "MPL-2.0",
"activity_log_id": 2170,
"scanner_result_url": "http://olympia.test/api/v5/scanner/results/124/"
"scanner_result_url": "http://olympia.test/api/v5/scanner/results/124/",
"file_original_hash": "sha256:3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c"
}
```

Expand Down
2 changes: 1 addition & 1 deletion src/olympia/devhub/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
log = olympia.core.logger.getLogger('z.devhub')


def process_validation(validation, file_hash=None, channel=amo.CHANNEL_LISTED):
def process_validation(validation, channel=amo.CHANNEL_LISTED):
"""Process validation results into the format expected by the web
frontend, including transforming certain fields into HTML, mangling
compatibility messages, and limiting the number of messages displayed."""
Expand Down
18 changes: 18 additions & 0 deletions src/olympia/files/migrations/0038_fileupload_original_hash.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 4.2.28 on 2026-02-24 14:13

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('files', '0037_fileupload_request_metadata'),
]

operations = [
migrations.AddField(
model_name='fileupload',
name='original_hash',
field=models.CharField(default='', max_length=255),
),
]
16 changes: 11 additions & 5 deletions src/olympia/files/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,10 @@ class STATUS_DISABLED_REASONS(EnumChoices):
upload_to=files_upload_to_callback,
)
size = models.PositiveIntegerField(default=0) # In bytes.
# The hash of the file (after repack and/or signing).
hash = models.CharField(max_length=255, default='')
# The original hash of the file, before we sign it, or repackage it in
# any other way.
# The original hash of the file uploaded by the developer, without any
# modifications.
original_hash = models.CharField(max_length=255, default='')
status = models.PositiveSmallIntegerField(
choices=STATUS_CHOICES.items(), default=amo.STATUS_AWAITING_REVIEW
Expand Down Expand Up @@ -189,8 +190,12 @@ def from_upload(cls, upload, version, parsed_data=None):
'is_mozilla_signed_extension', False
)
file_.is_signed = file_.is_mozilla_signed_extension
# This is the hash of the file uploaded by the developer, possibly
# repacked and/or signed.
file_.hash = upload.hash
file_.original_hash = file_.hash
# This is the hash of the file uploaded by the developer without any
# modifications (before repacking, signing, etc.).
file_.original_hash = upload.original_hash
file_.manifest_version = parsed_data.get('manifest_version')
log.info(f'New file: {file_!r} from {upload!r}')

Expand Down Expand Up @@ -428,6 +433,7 @@ class FileUpload(ModelBase):
max_length=255, default='', help_text="The user's original filename"
)
hash = models.CharField(max_length=255, default='')
original_hash = models.CharField(max_length=255, default='')
user = models.ForeignKey('users.UserProfile', on_delete=models.CASCADE)
valid = models.BooleanField(default=False)
validation = models.TextField(null=True)
Expand Down Expand Up @@ -512,6 +518,7 @@ def add_file(self, chunks, filename, size):
if hash_obj is None:
hash_obj = self.write_data_to_path(chunks)
self.hash = 'sha256:%s' % hash_obj.hexdigest()
self.original_hash = self.hash

# The following log statement is used by foxsec-pipeline.
log.info(
Expand Down Expand Up @@ -597,7 +604,7 @@ def processed_validation(self):

validation = self.load_validation()

return process_validation(validation, file_hash=self.hash)
return process_validation(validation)

@property
def passed_all_validations(self):
Expand Down Expand Up @@ -656,7 +663,6 @@ def processed_validation(self):

return process_validation(
json.loads(self.validation),
file_hash=self.file.original_hash,
channel=self.file.version.channel,
)

Expand Down
6 changes: 5 additions & 1 deletion src/olympia/files/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -868,7 +868,9 @@ def test_from_post_filename(self):

def test_from_post_hash(self):
hashdigest = hashlib.sha256(self.data).hexdigest()
assert self.upload().hash == 'sha256:%s' % hashdigest
upload = self.upload()
assert upload.hash == 'sha256:%s' % hashdigest
assert upload.original_hash == upload.hash

def test_from_post_is_one_query(self):
addon = Addon.objects.get(pk=3615)
Expand Down Expand Up @@ -1403,8 +1405,10 @@ def test_public_to_unreviewed(self):

def test_file_hash_copied_over(self):
upload = self.upload('webextension.xpi')
upload.original_hash = 'sha256:original'
file_ = File.from_upload(upload, self.version, parsed_data=self.parsed_data)
assert file_.hash == 'sha256:fake_hash'
assert file_.original_hash == 'sha256:original'

def test_extension_extension(self):
upload = self.upload('webextension.xpi')
Expand Down
2 changes: 2 additions & 0 deletions src/olympia/files/tests/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def test_repacking_xpi_files_with_mocks(
):
"""Opposite of test_not_repacking_non_xpi_files() (using same mocks)"""
upload = self.get_upload('webextension.xpi')
upload.update(original_hash='sha256:original')
get_sha256_mock.return_value = 'fakehashfrommock'
fake_results = {'errors': 0}
repack_fileupload(fake_results, upload.pk)
Expand All @@ -56,6 +57,7 @@ def test_repacking_xpi_files_with_mocks(
assert move_stored_file_mock.called
upload.reload()
assert upload.hash == 'sha256:fakehashfrommock'
assert upload.original_hash == 'sha256:original'

def test_repacking_xpi_files(self):
"""Test that repack_fileupload() does repack xpi files (no mocks)"""
Expand Down
1 change: 1 addition & 0 deletions src/olympia/versions/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,7 @@ def call_webhooks_on_source_code_uploaded(version_pk, activity_log_id):
),
'license_slug': version.license.slug,
'activity_log_id': activity_log_id,
'file_original_hash': version.file.original_hash,
}
call_webhooks(
event_name=WEBHOOK_ON_SOURCE_CODE_UPLOADED,
Expand Down
7 changes: 6 additions & 1 deletion src/olympia/versions/tests/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -879,14 +879,18 @@ def test_call_with_mock(self, call_webhooks_mock):
),
'license_slug': version.license.slug,
'activity_log_id': activity_log_id,
'file_original_hash': version.file.original_hash,
},
version=version,
)

@mock.patch('olympia.versions.tasks.call_webhooks')
def test_call_with_mock_and_deleted_version(self, call_webhooks_mock):
addon = addon_factory()
version = version_factory(addon=addon)
version = version_factory(
addon=addon,
file_kw={'original_hash': 'sha256:somehash'},
)
# Delete the version. The task uses `Version.unfiltered` to account for that.
version.delete()
activity_log_id = 123
Expand All @@ -905,6 +909,7 @@ def test_call_with_mock_and_deleted_version(self, call_webhooks_mock):
),
'license_slug': version.license.slug,
'activity_log_id': activity_log_id,
'file_original_hash': version.file.original_hash,
},
version=version,
)
Expand Down
Loading