diff --git a/docs/topics/development/scanner_pipeline.md b/docs/topics/development/scanner_pipeline.md index dee81b5c7d92..ef9605f730ad 100644 --- a/docs/topics/development/scanner_pipeline.md +++ b/docs/topics/development/scanner_pipeline.md @@ -73,7 +73,8 @@ The payload sent looks like this: "download_source_url": "http://olympia.test/downloads/source/42", "license_slug": "MPL-2.0", "activity_log_id": 2170, - "scanner_result_url": "http://olympia.test/api/v5/scanner/results/124/" + "scanner_result_url": "http://olympia.test/api/v5/scanner/results/124/", + "file_original_hash": "sha256:3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c" } ``` diff --git a/src/olympia/devhub/utils.py b/src/olympia/devhub/utils.py index 5993cd711b98..dcadb423fb3e 100644 --- a/src/olympia/devhub/utils.py +++ b/src/olympia/devhub/utils.py @@ -31,7 +31,7 @@ log = olympia.core.logger.getLogger('z.devhub') -def process_validation(validation, file_hash=None, channel=amo.CHANNEL_LISTED): +def process_validation(validation, channel=amo.CHANNEL_LISTED): """Process validation results into the format expected by the web frontend, including transforming certain fields into HTML, mangling compatibility messages, and limiting the number of messages displayed.""" diff --git a/src/olympia/files/migrations/0038_fileupload_original_hash.py b/src/olympia/files/migrations/0038_fileupload_original_hash.py new file mode 100644 index 000000000000..971dac504c21 --- /dev/null +++ b/src/olympia/files/migrations/0038_fileupload_original_hash.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.28 on 2026-02-24 14:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('files', '0037_fileupload_request_metadata'), + ] + + operations = [ + migrations.AddField( + model_name='fileupload', + name='original_hash', + field=models.CharField(default='', max_length=255), + ), + ] diff --git a/src/olympia/files/models.py b/src/olympia/files/models.py index c02f7332c248..e8a89572138c 100644 --- a/src/olympia/files/models.py +++ b/src/olympia/files/models.py @@ -107,9 +107,10 @@ class STATUS_DISABLED_REASONS(EnumChoices): upload_to=files_upload_to_callback, ) size = models.PositiveIntegerField(default=0) # In bytes. + # The hash of the file (after repack and/or signing). hash = models.CharField(max_length=255, default='') - # The original hash of the file, before we sign it, or repackage it in - # any other way. + # The original hash of the file uploaded by the developer, without any + # modifications. original_hash = models.CharField(max_length=255, default='') status = models.PositiveSmallIntegerField( choices=STATUS_CHOICES.items(), default=amo.STATUS_AWAITING_REVIEW @@ -189,8 +190,12 @@ def from_upload(cls, upload, version, parsed_data=None): 'is_mozilla_signed_extension', False ) file_.is_signed = file_.is_mozilla_signed_extension + # This is the hash of the file uploaded by the developer, possibly + # repacked and/or signed. file_.hash = upload.hash - file_.original_hash = file_.hash + # This is the hash of the file uploaded by the developer without any + # modifications (before repacking, signing, etc.). + file_.original_hash = upload.original_hash file_.manifest_version = parsed_data.get('manifest_version') log.info(f'New file: {file_!r} from {upload!r}') @@ -428,6 +433,7 @@ class FileUpload(ModelBase): max_length=255, default='', help_text="The user's original filename" ) hash = models.CharField(max_length=255, default='') + original_hash = models.CharField(max_length=255, default='') user = models.ForeignKey('users.UserProfile', on_delete=models.CASCADE) valid = models.BooleanField(default=False) validation = models.TextField(null=True) @@ -512,6 +518,7 @@ def add_file(self, chunks, filename, size): if hash_obj is None: hash_obj = self.write_data_to_path(chunks) self.hash = 'sha256:%s' % hash_obj.hexdigest() + self.original_hash = self.hash # The following log statement is used by foxsec-pipeline. log.info( @@ -597,7 +604,7 @@ def processed_validation(self): validation = self.load_validation() - return process_validation(validation, file_hash=self.hash) + return process_validation(validation) @property def passed_all_validations(self): @@ -656,7 +663,6 @@ def processed_validation(self): return process_validation( json.loads(self.validation), - file_hash=self.file.original_hash, channel=self.file.version.channel, ) diff --git a/src/olympia/files/tests/test_models.py b/src/olympia/files/tests/test_models.py index 5a84f6c678b9..b12d4397b3c0 100644 --- a/src/olympia/files/tests/test_models.py +++ b/src/olympia/files/tests/test_models.py @@ -868,7 +868,9 @@ def test_from_post_filename(self): def test_from_post_hash(self): hashdigest = hashlib.sha256(self.data).hexdigest() - assert self.upload().hash == 'sha256:%s' % hashdigest + upload = self.upload() + assert upload.hash == 'sha256:%s' % hashdigest + assert upload.original_hash == upload.hash def test_from_post_is_one_query(self): addon = Addon.objects.get(pk=3615) @@ -1403,8 +1405,10 @@ def test_public_to_unreviewed(self): def test_file_hash_copied_over(self): upload = self.upload('webextension.xpi') + upload.original_hash = 'sha256:original' file_ = File.from_upload(upload, self.version, parsed_data=self.parsed_data) assert file_.hash == 'sha256:fake_hash' + assert file_.original_hash == 'sha256:original' def test_extension_extension(self): upload = self.upload('webextension.xpi') diff --git a/src/olympia/files/tests/test_tasks.py b/src/olympia/files/tests/test_tasks.py index c79ff9c94d0d..f501464b0fdd 100644 --- a/src/olympia/files/tests/test_tasks.py +++ b/src/olympia/files/tests/test_tasks.py @@ -44,6 +44,7 @@ def test_repacking_xpi_files_with_mocks( ): """Opposite of test_not_repacking_non_xpi_files() (using same mocks)""" upload = self.get_upload('webextension.xpi') + upload.update(original_hash='sha256:original') get_sha256_mock.return_value = 'fakehashfrommock' fake_results = {'errors': 0} repack_fileupload(fake_results, upload.pk) @@ -56,6 +57,7 @@ def test_repacking_xpi_files_with_mocks( assert move_stored_file_mock.called upload.reload() assert upload.hash == 'sha256:fakehashfrommock' + assert upload.original_hash == 'sha256:original' def test_repacking_xpi_files(self): """Test that repack_fileupload() does repack xpi files (no mocks)""" diff --git a/src/olympia/versions/tasks.py b/src/olympia/versions/tasks.py index 09ea21257714..1e070a831742 100644 --- a/src/olympia/versions/tasks.py +++ b/src/olympia/versions/tasks.py @@ -452,6 +452,7 @@ def call_webhooks_on_source_code_uploaded(version_pk, activity_log_id): ), 'license_slug': version.license.slug, 'activity_log_id': activity_log_id, + 'file_original_hash': version.file.original_hash, } call_webhooks( event_name=WEBHOOK_ON_SOURCE_CODE_UPLOADED, diff --git a/src/olympia/versions/tests/test_tasks.py b/src/olympia/versions/tests/test_tasks.py index 2ddb5e0679fa..4e5e9e833944 100644 --- a/src/olympia/versions/tests/test_tasks.py +++ b/src/olympia/versions/tests/test_tasks.py @@ -879,6 +879,7 @@ def test_call_with_mock(self, call_webhooks_mock): ), 'license_slug': version.license.slug, 'activity_log_id': activity_log_id, + 'file_original_hash': version.file.original_hash, }, version=version, ) @@ -886,7 +887,10 @@ def test_call_with_mock(self, call_webhooks_mock): @mock.patch('olympia.versions.tasks.call_webhooks') def test_call_with_mock_and_deleted_version(self, call_webhooks_mock): addon = addon_factory() - version = version_factory(addon=addon) + version = version_factory( + addon=addon, + file_kw={'original_hash': 'sha256:somehash'}, + ) # Delete the version. The task uses `Version.unfiltered` to account for that. version.delete() activity_log_id = 123 @@ -905,6 +909,7 @@ def test_call_with_mock_and_deleted_version(self, call_webhooks_mock): ), 'license_slug': version.license.slug, 'activity_log_id': activity_log_id, + 'file_original_hash': version.file.original_hash, }, version=version, )