From 8699bc15bc8a7cfa91d692e5e934b1c0821b402d Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 10 Nov 2025 10:19:34 -0500 Subject: [PATCH 01/36] Rework mime type white list --- src/onegov/form/validators.py | 44 ++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index e84307d6d7..6818a061df 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -129,17 +129,49 @@ class WhitelistedMimeType: """ whitelist: Collection[str] = { - 'application/excel', - 'application/vnd.ms-excel', - 'application/msword', + # documents + 'application/msword', # doc 'application/pdf', + 'application/rtf', + 'application/vnd.ms-excel', # xls + ('application/vnd.openxmlformats-officedocument.' + 'presentationml.presentation'), # pptx + ('application/vnd.openxmlformats-officedocument.' + 'spreadsheetml.sheet'), # xlsx + ('application/vnd.openxmlformats-officedocument.' + 'wordprocessingml.document'), # docx + + # archives 'application/zip', + + # text / data + 'text/csv', + 'text/plain', + + # images + 'image/bmp', 'image/gif', - 'image/jpeg', + 'image/jpeg', # jpeg, jpg 'image/png', + 'image/svg', + 'image/svg+xml', + 'image/tiff', + 'image/webp', # shall we allow it? 'image/x-ms-bmp', - 'text/plain', - 'text/csv' + + # audio + 'audio/mp4', + 'audio/mpeg', + 'audio/wav', + 'audio/webm', # weba + + # video + 'video/mp4', + 'video/mpeg', # mpg, mpeg + 'video/ogg', + 'video/quicktime', # mov + 'video/webm', # webm + 'video/x-msvideo', # avi } message = _('Files of this type are not supported.') From 66ac4043a279d1353ceebda34865618cfc403d9d Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 17 Nov 2025 11:00:03 -0500 Subject: [PATCH 02/36] Ensure mime type validator for file fields in formcode --- tests/onegov/form/test_parser.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index 4f23ff3378..1a65e206c0 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -10,7 +10,7 @@ from onegov.form.fields import ( DateTimeLocalField, MultiCheckboxField, TimeField, URLField, VideoURLField) from onegov.form.parser.grammar import field_help_identifier -from onegov.form.validators import LaxDataRequired +from onegov.form.validators import LaxDataRequired, WhitelistedMimeType from onegov.form.validators import ValidDateRange from onegov.pay import Price from textwrap import dedent @@ -341,6 +341,16 @@ def test_parse_fileinput() -> None: assert isinstance(form['file'], FileField) assert form['file'].widget.multiple is False # type: ignore[attr-defined] + # verify mime type validator + field = form._fields['file'] + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + + form = parse_form("File = *.*")() + field = form._fields['file'] + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + def test_parse_multiplefileinput() -> None: form = parse_form("Files = *.pdf|*.doc (multiple)")() @@ -349,6 +359,17 @@ def test_parse_multiplefileinput() -> None: assert isinstance(form['files'], FileField) assert form['files'].widget.multiple is True # type: ignore[attr-defined] + # verify mime type validator + field = form._fields['files'] + assert field.validators + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + + form = parse_form("File = *.*")() + field = form._fields['files'] + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + def test_parse_radio() -> None: From 5ebe0fbe06214dc2c00f1690ffd29a46468b7dd0 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 20 Nov 2025 07:21:36 -0500 Subject: [PATCH 03/36] Adds mime type validator by default --- .../election_day/forms/upload/common.py | 4 +- src/onegov/form/fields.py | 4 +- src/onegov/form/validators.py | 13 +++- tests/onegov/form/test_fields.py | 68 +++++++++++++++++-- tests/onegov/form/test_parser.py | 30 ++++---- 5 files changed, 99 insertions(+), 20 deletions(-) diff --git a/src/onegov/election_day/forms/upload/common.py b/src/onegov/election_day/forms/upload/common.py index e50464640e..2a527194b8 100644 --- a/src/onegov/election_day/forms/upload/common.py +++ b/src/onegov/election_day/forms/upload/common.py @@ -11,8 +11,8 @@ 'text/csv' } ALLOWED_MIME_TYPES_XML = { - 'application/xml', - 'text/xml', + 'application/xml', # official, standard + 'text/xml', # deprecated MIME type for XML content 'text/plain' } diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 8a6aea3e63..449c6b330b 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -20,7 +20,7 @@ from onegov.file.utils import IMAGE_MIME_TYPES_AND_SVG from onegov.form import log, _ from onegov.form.utils import path_to_filename -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ValidPhoneNumber, WhitelistedMimeType from onegov.form.widgets import ChosenSelectWidget from onegov.form.widgets import LinkPanelWidget from onegov.form.widgets import DurationInput @@ -260,6 +260,7 @@ class UploadField(FileField): action: Literal['keep', 'replace', 'delete'] file: IO[bytes] | None filename: str | None + validators = [WhitelistedMimeType()] if TYPE_CHECKING: def __init__( @@ -448,6 +449,7 @@ def _add_entry(self, d: _MultiDictLikeWithGetlist, /) -> UploadField: upload_field_class: type[UploadField] = UploadField upload_widget: Widget[UploadField] = UploadWidget() + validators = [WhitelistedMimeType()] def __init__( self, diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 6818a061df..005f62984a 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -141,6 +141,9 @@ class WhitelistedMimeType: ('application/vnd.openxmlformats-officedocument.' 'wordprocessingml.document'), # docx + # xml + 'application/xml', + # archives 'application/zip', @@ -184,7 +187,15 @@ def __call__(self, form: Form, field: Field) -> None: if not field.data: return - if field.data['mimetype'] not in self.whitelist: + if isinstance(field.data, list): # UploadMultipleField + for data in field.data: + if not data: + continue # in case of file deletion + + if data['mimetype'] not in self.whitelist: + raise ValidationError(field.gettext(self.message)) + + elif field.data['mimetype'] not in self.whitelist: raise ValidationError(field.gettext(self.message)) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index d3dfc55984..b04cc56637 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -22,15 +22,19 @@ from onegov.form.fields import UploadField from onegov.form.fields import UploadMultipleField from onegov.form.fields import URLField -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ( + ValidPhoneNumber, WhitelistedMimeType, ExpectedExtensions) from unittest.mock import patch from wtforms.validators import Optional from wtforms.validators import URL -from typing import Any, TYPE_CHECKING +from typing import Any, TYPE_CHECKING, Self + if TYPE_CHECKING: from webob.request import _FieldStorageWithFile + from onegov.form.types import ( + FormT, Validators) class DummyPostData(dict[str, Any]): @@ -56,9 +60,11 @@ def create_file( def test_upload_field() -> None: - def create_field() -> tuple[Form, UploadField]: + def create_field( + validators: Validators[FormT, Self] | None = None + ) -> tuple[Form, UploadField]: form = Form() - field = UploadField() + field = UploadField(validators=validators) field = field.bind(form, 'upload') # type: ignore[attr-defined] return form, field @@ -68,12 +74,17 @@ def create_field() -> tuple[Form, UploadField]: assert data == {} assert field.file is None assert field.filename is None + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() data = field.process_fieldstorage('') assert data == {} assert field.file is None assert field.filename is None + assert field.validate(form) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) textfile = create_file('text/plain', 'foo.txt', b'foo') data = field.process_fieldstorage(textfile) @@ -84,6 +95,9 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(data) == b'foo' # type: ignore[arg-type] assert field.filename == 'foo.txt' assert field.file.read() == b'foo' # type: ignore[attr-defined] + assert field.validate(form) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() textfile = create_file('text/plain', 'C:/mydata/bar.txt', b'bar') @@ -95,6 +109,20 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(data) == b'bar' # type: ignore[arg-type] assert field.filename == 'bar.txt' assert field.file.read() == b'bar' # type: ignore[union-attr] + assert field.validate(form) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + + # failing mime type validator + form, field = create_field(validators=[ExpectedExtensions(['.pdf'])]) + textfile = create_file('text/plain', 'baz.txt', b'baz') + field.data = field.process_fieldstorage(textfile) + assert field.data['filename'] == 'baz.txt' + assert field.data['mimetype'] == 'text/plain' + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert not field.validate(form) + assert 'Files of this type are not supported.' in field.errors # Test rendering form, field = create_field() @@ -104,6 +132,8 @@ def create_field() -> tuple[Form, UploadField]: field.data = field.process_fieldstorage(textfile) assert 'without-data' in field(force_simple=True) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) html = field() assert 'with-data' in html @@ -111,6 +141,8 @@ def create_field() -> tuple[Form, UploadField]: assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' not in html + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) html = field(resend_upload=True) assert 'with-data' in html @@ -118,12 +150,16 @@ def create_field() -> tuple[Form, UploadField]: assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' in html + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # Test submit form, field = create_field() field.process(DummyPostData({})) assert field.validate(form) assert field.data == {} + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() field.process(DummyPostData({'upload': 'abcd'})) @@ -132,6 +168,8 @@ def create_field() -> tuple[Form, UploadField]: assert field.data == {} assert field.file is None assert field.filename is None + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # ... simple form, field = create_field() @@ -146,6 +184,8 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # ... with select form, field = create_field() @@ -153,6 +193,8 @@ def create_field() -> tuple[Form, UploadField]: field.process(DummyPostData({'upload': ['keep', textfile]})) assert field.validate(form) assert field.action == 'keep' + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -160,6 +202,8 @@ def create_field() -> tuple[Form, UploadField]: assert field.validate(form) assert field.action == 'delete' assert field.data == {} + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -173,6 +217,8 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # ... with select and keep upload previous = field.data @@ -191,6 +237,8 @@ def create_field() -> tuple[Form, UploadField]: assert field.data['mimetype'] == 'text/plain' assert field.data['size'] == 6 assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) field.process(DummyPostData({'upload': [ 'delete', @@ -203,6 +251,8 @@ def create_field() -> tuple[Form, UploadField]: assert field2.validate(form) assert field2.action == 'delete' assert field2.data == {} + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) field.process(DummyPostData({'upload': [ 'replace', @@ -221,6 +271,8 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(field2.data) == b'foobaz' # type: ignore[arg-type] assert field2.filename == 'foobaz.txt' assert field2.file.read() == b'foobaz' # type: ignore[union-attr] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) def test_upload_multiple_field() -> None: @@ -245,6 +297,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: file1 = create_file('text/plain', 'baz.txt', b'baz') file2 = create_file('text/plain', 'foobar.txt', b'foobar') field.process(DummyPostData({'uploads': [file1, file2]})) + assert field.validate(form) assert len(field.data) == 2 assert field.data[0]['filename'] == 'baz.txt' assert field.data[0]['mimetype'] == 'text/plain' @@ -266,7 +319,12 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert file_field2.filename == 'foobar.txt' assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] + # verify attached validators + assert field.validators + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + html = field(force_simple=True) + assert field.validate(form) assert 'without-data' in html assert 'multiple' in html assert 'name="uploads"' in html @@ -274,6 +332,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert 'name="uploads-0"' not in html html = field() + assert field.validate(form) assert 'with-data' in html assert 'name="uploads-0"' in html assert 'Uploaded file: baz.txt (3 Bytes) ✓' in html @@ -290,6 +349,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert 'multiple' in html html = field(resend_upload=True) + assert field.validate(form) assert 'with-data' in html assert 'Uploaded file: baz.txt (3 Bytes) ✓' in html assert 'Uploaded file: foobar.txt (6 Bytes) ✓' in html diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index 1a65e206c0..c1c1fac028 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -334,6 +334,10 @@ def test_parse_time() -> None: assert isinstance(form['time'], TimeField) +def _find_validator(field, cls): + return next((v for v in field.validators if isinstance(v, cls)), None) + + def test_parse_fileinput() -> None: form = parse_form("File = *.pdf|*.doc")() @@ -341,10 +345,12 @@ def test_parse_fileinput() -> None: assert isinstance(form['file'], FileField) assert form['file'].widget.multiple is False # type: ignore[attr-defined] - # verify mime type validator - field = form._fields['file'] - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + # verify attached mime type validator + assert form['file'].validators + validator = _find_validator(form['file'], WhitelistedMimeType) + assert validator.whitelist == { + 'application/msword', 'application/pdf' + } form = parse_form("File = *.*")() field = form._fields['file'] @@ -354,25 +360,25 @@ def test_parse_fileinput() -> None: def test_parse_multiplefileinput() -> None: form = parse_form("Files = *.pdf|*.doc (multiple)")() + # form = parse_form("Files = *.pdf (multiple)")() assert form['files'].label.text == 'Files' assert isinstance(form['files'], FileField) assert form['files'].widget.multiple is True # type: ignore[attr-defined] - # verify mime type validator - field = form._fields['files'] - assert field.validators - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + # verify attached mime type validator + validator = _find_validator(form['files'], WhitelistedMimeType) + assert validator.whitelist == { + 'application/msword', 'application/pdf' + } - form = parse_form("File = *.*")() + form = parse_form("Files = *.* (multiple)")() field = form._fields['files'] assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + assert field.validators[0].whitelist == WhitelistedMimeType.whitelist def test_parse_radio() -> None: - text = dedent(""" Gender = ( ) Male From 7eed62a9be34df6f0c9295a086f752b718340ee7 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 10:17:31 +0100 Subject: [PATCH 04/36] Fix wrongly attached validators --- src/onegov/form/fields.py | 1 + tests/onegov/form/test_parser.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 449c6b330b..2550ef800d 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -498,6 +498,7 @@ def __init__( widget=widget, # type:ignore[arg-type] render_kw=render_kw, name=name, + validators=validators, _form=_form, _prefix=_prefix, _translations=_translations, diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index c1c1fac028..aeeef4b538 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -373,9 +373,8 @@ def test_parse_multiplefileinput() -> None: } form = parse_form("Files = *.* (multiple)")() - field = form._fields['files'] - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[0].whitelist == WhitelistedMimeType.whitelist + validator = _find_validator(form['files'], WhitelistedMimeType) + assert validator.whitelist == WhitelistedMimeType.whitelist def test_parse_radio() -> None: From 4ab42dff8a842408ca722524849937031ec1ce1f Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 10:48:54 +0100 Subject: [PATCH 05/36] Revert --- src/onegov/form/fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 2550ef800d..32c4a454bb 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -481,7 +481,6 @@ def __init__( # a lot of the arguments we just pass through to the subfield unbound_field = self.upload_field_class( - validators=validators, # type:ignore[arg-type] filters=filters, description=description, widget=upload_widget, From cf252e2df4a4ec4f42eb8132a1bd309092478c94 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 11:05:32 +0100 Subject: [PATCH 06/36] improve tests and fix linter issues --- tests/onegov/form/test_parser.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index aeeef4b538..b966fce479 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -4,6 +4,9 @@ from dateutil.relativedelta import relativedelta from decimal import Decimal + +from wtforms import Field + from onegov.form import Form, errors, find_field from onegov.form import parse_formcode, parse_form, flatten_fieldsets from onegov.form.errors import InvalidIndentSyntax @@ -25,10 +28,13 @@ from wtforms.validators import Regexp -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + if TYPE_CHECKING: from pyparsing import ParserElement, ParseResults + from onegov.form.types import Validator + def parse(expr: ParserElement, text: str) -> ParseResults: return expr.parseString(text) @@ -334,7 +340,10 @@ def test_parse_time() -> None: assert isinstance(form['time'], TimeField) -def _find_validator(field, cls): +def _find_validator( + field: Field | FileField, + cls: type +) -> Validator[Any, Any] | None: return next((v for v in field.validators if isinstance(v, cls)), None) @@ -348,19 +357,19 @@ def test_parse_fileinput() -> None: # verify attached mime type validator assert form['file'].validators validator = _find_validator(form['file'], WhitelistedMimeType) - assert validator.whitelist == { + assert validator + assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("File = *.*")() - field = form._fields['file'] - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + validator = _find_validator(form['file'], WhitelistedMimeType) + assert validator + assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] def test_parse_multiplefileinput() -> None: form = parse_form("Files = *.pdf|*.doc (multiple)")() - # form = parse_form("Files = *.pdf (multiple)")() assert form['files'].label.text == 'Files' assert isinstance(form['files'], FileField) @@ -368,13 +377,15 @@ def test_parse_multiplefileinput() -> None: # verify attached mime type validator validator = _find_validator(form['files'], WhitelistedMimeType) - assert validator.whitelist == { + assert validator + assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("Files = *.* (multiple)")() validator = _find_validator(form['files'], WhitelistedMimeType) - assert validator.whitelist == WhitelistedMimeType.whitelist + assert validator + assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] def test_parse_radio() -> None: From 6e9afd6280a0dfdfe10b42ac1874491b0dbe0f5e Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 14:05:00 +0100 Subject: [PATCH 07/36] Rework tests --- tests/onegov/form/test_fields.py | 82 ++++++++++++++++---------------- tests/onegov/form/test_parser.py | 29 ++++++----- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index b04cc56637..2b37849a9e 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -6,6 +6,7 @@ from cgi import FieldStorage from copy import deepcopy from datetime import datetime + from onegov.core.utils import Bunch from onegov.core.utils import dictionary_to_binary from onegov.form import Form @@ -25,6 +26,7 @@ from onegov.form.validators import ( ValidPhoneNumber, WhitelistedMimeType, ExpectedExtensions) from unittest.mock import patch +from wtforms import FileField, Field from wtforms.validators import Optional from wtforms.validators import URL @@ -34,7 +36,23 @@ if TYPE_CHECKING: from webob.request import _FieldStorageWithFile from onegov.form.types import ( - FormT, Validators) + FormT, Validators, Validator) + + +def assert_whitelisted_mimetype_validator( + field: UploadField | UploadMultipleField +) -> None: + + validator = find_validator(field, WhitelistedMimeType) + assert validator + assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] + + +def find_validator( + field: Field | FileField, + cls: type +) -> Validator[Any, Any] | None: + return next((v for v in field.validators if isinstance(v, cls)), None) class DummyPostData(dict[str, Any]): @@ -74,8 +92,7 @@ def create_field( assert data == {} assert field.file is None assert field.filename is None - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() data = field.process_fieldstorage('') @@ -83,8 +100,7 @@ def create_field( assert field.file is None assert field.filename is None assert field.validate(form) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) textfile = create_file('text/plain', 'foo.txt', b'foo') data = field.process_fieldstorage(textfile) @@ -96,8 +112,7 @@ def create_field( assert field.filename == 'foo.txt' assert field.file.read() == b'foo' # type: ignore[attr-defined] assert field.validate(form) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() textfile = create_file('text/plain', 'C:/mydata/bar.txt', b'bar') @@ -110,17 +125,17 @@ def create_field( assert field.filename == 'bar.txt' assert field.file.read() == b'bar' # type: ignore[union-attr] assert field.validate(form) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # failing mime type validator form, field = create_field(validators=[ExpectedExtensions(['.pdf'])]) textfile = create_file('text/plain', 'baz.txt', b'baz') - field.data = field.process_fieldstorage(textfile) - assert field.data['filename'] == 'baz.txt' - assert field.data['mimetype'] == 'text/plain' - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + data = field.data = field.process_fieldstorage(textfile) + assert data['filename'] == 'baz.txt' + assert data['mimetype'] == 'text/plain' + validator = find_validator(field, WhitelistedMimeType) + assert validator + assert validator.whitelist == {'application/pdf'} # type:ignore[attr-defined] assert not field.validate(form) assert 'Files of this type are not supported.' in field.errors @@ -132,8 +147,7 @@ def create_field( field.data = field.process_fieldstorage(textfile) assert 'without-data' in field(force_simple=True) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) html = field() assert 'with-data' in html @@ -141,8 +155,7 @@ def create_field( assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' not in html - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) html = field(resend_upload=True) assert 'with-data' in html @@ -150,16 +163,14 @@ def create_field( assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' in html - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # Test submit form, field = create_field() field.process(DummyPostData({})) assert field.validate(form) assert field.data == {} - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() field.process(DummyPostData({'upload': 'abcd'})) @@ -168,8 +179,7 @@ def create_field( assert field.data == {} assert field.file is None assert field.filename is None - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # ... simple form, field = create_field() @@ -184,8 +194,7 @@ def create_field( assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # ... with select form, field = create_field() @@ -193,8 +202,7 @@ def create_field( field.process(DummyPostData({'upload': ['keep', textfile]})) assert field.validate(form) assert field.action == 'keep' - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -202,8 +210,7 @@ def create_field( assert field.validate(form) assert field.action == 'delete' assert field.data == {} - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -217,8 +224,7 @@ def create_field( assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # ... with select and keep upload previous = field.data @@ -237,8 +243,7 @@ def create_field( assert field.data['mimetype'] == 'text/plain' assert field.data['size'] == 6 assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) field.process(DummyPostData({'upload': [ 'delete', @@ -251,8 +256,7 @@ def create_field( assert field2.validate(form) assert field2.action == 'delete' assert field2.data == {} - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) field.process(DummyPostData({'upload': [ 'replace', @@ -271,8 +275,7 @@ def create_field( assert dictionary_to_binary(field2.data) == b'foobaz' # type: ignore[arg-type] assert field2.filename == 'foobaz.txt' assert field2.file.read() == b'foobaz' # type: ignore[union-attr] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) def test_upload_multiple_field() -> None: @@ -320,8 +323,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] # verify attached validators - assert field.validators - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert_whitelisted_mimetype_validator(field) html = field(force_simple=True) assert field.validate(form) diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index b966fce479..6acea95e59 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -11,7 +11,12 @@ from onegov.form import parse_formcode, parse_form, flatten_fieldsets from onegov.form.errors import InvalidIndentSyntax from onegov.form.fields import ( - DateTimeLocalField, MultiCheckboxField, TimeField, URLField, VideoURLField) + DateTimeLocalField, + MultiCheckboxField, + TimeField, + URLField, + VideoURLField, +) from onegov.form.parser.grammar import field_help_identifier from onegov.form.validators import LaxDataRequired, WhitelistedMimeType from onegov.form.validators import ValidDateRange @@ -40,6 +45,13 @@ def parse(expr: ParserElement, text: str) -> ParseResults: return expr.parseString(text) +def find_validator( + field: Field | FileField, + cls: type +) -> Validator[Any, Any] | None: + return next((v for v in field.validators if isinstance(v, cls)), None) + + @pytest.mark.parametrize('comment,output', [ ('<< Some text >>', 'Some text'), ('<< [Z](www.co.me) >>', '[Z](www.co.me)') @@ -340,13 +352,6 @@ def test_parse_time() -> None: assert isinstance(form['time'], TimeField) -def _find_validator( - field: Field | FileField, - cls: type -) -> Validator[Any, Any] | None: - return next((v for v in field.validators if isinstance(v, cls)), None) - - def test_parse_fileinput() -> None: form = parse_form("File = *.pdf|*.doc")() @@ -356,14 +361,14 @@ def test_parse_fileinput() -> None: # verify attached mime type validator assert form['file'].validators - validator = _find_validator(form['file'], WhitelistedMimeType) + validator = find_validator(form['file'], WhitelistedMimeType) assert validator assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("File = *.*")() - validator = _find_validator(form['file'], WhitelistedMimeType) + validator = find_validator(form['file'], WhitelistedMimeType) assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] @@ -376,14 +381,14 @@ def test_parse_multiplefileinput() -> None: assert form['files'].widget.multiple is True # type: ignore[attr-defined] # verify attached mime type validator - validator = _find_validator(form['files'], WhitelistedMimeType) + validator = find_validator(form['files'], WhitelistedMimeType) assert validator assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("Files = *.* (multiple)")() - validator = _find_validator(form['files'], WhitelistedMimeType) + validator = find_validator(form['files'], WhitelistedMimeType) assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] From 3f718d2612866f77e1c9a01eeeb44a2c3f15626b Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 14:09:46 +0100 Subject: [PATCH 08/36] Fix file size validator and align --- src/onegov/form/validators.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 005f62984a..a9dad595cb 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -31,7 +31,8 @@ from wtforms.validators import ValidationError -from typing import Generic, TYPE_CHECKING +from typing import Generic, TYPE_CHECKING, Any + if TYPE_CHECKING: from collections.abc import Collection, Sequence from onegov.core.orm import Base @@ -114,7 +115,18 @@ def __call__(self, form: Form, field: Field) -> None: if not field.data: return - if field.data.get('size', 0) > self.max_bytes: + if isinstance(field.data, list): # UploadMultipleField + for data in field.data: + if not data: + continue # in case of file deletion + + self.validate_filesize(field, data) + + else: + self.validate_filesize(field, field.data) + + def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: + if data.get('size', 0) > self.max_bytes: message = field.gettext(self.message).format( humanize.naturalsize(self.max_bytes) ) @@ -192,11 +204,15 @@ def __call__(self, form: Form, field: Field) -> None: if not data: continue # in case of file deletion - if data['mimetype'] not in self.whitelist: - raise ValidationError(field.gettext(self.message)) + self.validate_mimetype(field, data) + + else: + self.validate_mimetype(field, field.data) - elif field.data['mimetype'] not in self.whitelist: - raise ValidationError(field.gettext(self.message)) + def validate_mimetype(self, field: Field, data: dict[Any, Any]) -> None: + if data['mimetype'] not in self.whitelist: + message = field.gettext(self.message) + raise ValidationError(field.gettext(message)) class ExpectedExtensions(WhitelistedMimeType): From 557202052ffd71071ec7b08a1fd2d3791fb38298 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Tue, 2 Dec 2025 09:20:53 +0100 Subject: [PATCH 09/36] Set mime types for all upload fields --- src/onegov/agency/forms/agency.py | 7 +- src/onegov/election_day/forms/election.py | 4 +- .../election_day/forms/election_compound.py | 8 +- src/onegov/election_day/forms/vote.py | 4 +- src/onegov/form/validators.py | 117 +++++++++++------- src/onegov/landsgemeinde/forms/agenda.py | 6 +- src/onegov/landsgemeinde/forms/assembly.py | 14 +-- src/onegov/org/forms/directory.py | 2 +- src/onegov/org/forms/event.py | 14 +-- src/onegov/org/forms/parliamentarian.py | 5 +- src/onegov/pas/forms/data_import.py | 4 + src/onegov/pas/forms/parliamentarian.py | 5 +- .../forms/accreditation.py | 24 ++-- .../translator_directory/forms/mutation.py | 24 ++-- 14 files changed, 129 insertions(+), 109 deletions(-) diff --git a/src/onegov/agency/forms/agency.py b/src/onegov/agency/forms/agency.py index b80029ee0a..8dc860f930 100644 --- a/src/onegov/agency/forms/agency.py +++ b/src/onegov/agency/forms/agency.py @@ -16,7 +16,7 @@ from onegov.form.fields import ChosenSelectField, HtmlField from onegov.form.fields import MultiCheckboxField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_IMAGE from onegov.form.validators import WhitelistedMimeType from onegov.gis import CoordinatesField from sqlalchemy import func @@ -73,10 +73,7 @@ class ExtendedAgencyForm(Form): organigram = UploadField( label=_('Organigram'), validators=[ - WhitelistedMimeType({ - 'image/jpeg', - 'image/png', - }), + WhitelistedMimeType(MIME_TYPES_IMAGE), FileSizeLimit(1 * 1024 * 1024) ] ) diff --git a/src/onegov/election_day/forms/election.py b/src/onegov/election_day/forms/election.py index 2c720e4933..9d4ff54478 100644 --- a/src/onegov/election_day/forms/election.py +++ b/src/onegov/election_day/forms/election.py @@ -13,7 +13,7 @@ from onegov.form.fields import ChosenSelectMultipleField from onegov.form.fields import PanelField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import WhitelistedMimeType from re import findall from sqlalchemy import or_ @@ -323,7 +323,7 @@ class ElectionForm(Form): explanations_pdf = UploadField( label=_('Explanations (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link') diff --git a/src/onegov/election_day/forms/election_compound.py b/src/onegov/election_day/forms/election_compound.py index 61be3c5e4c..76db82f6e8 100644 --- a/src/onegov/election_day/forms/election_compound.py +++ b/src/onegov/election_day/forms/election_compound.py @@ -12,7 +12,7 @@ from onegov.form.fields import ChosenSelectMultipleField from onegov.form.fields import PanelField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import WhitelistedMimeType from re import findall from sqlalchemy import or_ @@ -228,7 +228,7 @@ class ElectionCompoundForm(Form): explanations_pdf = UploadField( label=_('Explanations (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link') @@ -237,7 +237,7 @@ class ElectionCompoundForm(Form): upper_apportionment_pdf = UploadField( label=_('Upper apportionment (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link'), @@ -247,7 +247,7 @@ class ElectionCompoundForm(Form): lower_apportionment_pdf = UploadField( label=_('Lower apportionment (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link'), diff --git a/src/onegov/election_day/forms/vote.py b/src/onegov/election_day/forms/vote.py index 1719e07baa..1b5df4a516 100644 --- a/src/onegov/election_day/forms/vote.py +++ b/src/onegov/election_day/forms/vote.py @@ -10,7 +10,7 @@ from onegov.form.fields import ChosenSelectField from onegov.form.fields import PanelField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import WhitelistedMimeType from wtforms.fields import BooleanField from wtforms.fields import DateField @@ -280,7 +280,7 @@ class VoteForm(Form): explanations_pdf = UploadField( label=_('Explanations (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link') diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index a9dad595cb..5056747e44 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -133,6 +133,70 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: raise ValidationError(message) +MIME_TYPES_PDF = { + 'application/pdf', +} + +# for now not allowed by default +MIME_TYPES_JSON = { + 'application/json', +} + +MIME_TYPES_DOCUMENT = { + 'application/msword', # doc + 'application/rtf', + *MIME_TYPES_PDF, + 'application/vnd.ms-excel', # xls + ('application/vnd.openxmlformats-officedocument.' + 'presentationml.presentation'), # pptx + ('application/vnd.openxmlformats-officedocument.' + 'spreadsheetml.sheet'), # xlsx + ('application/vnd.openxmlformats-officedocument.' + 'wordprocessingml.document'), # docx +} + +MIME_TYPES_XML = { + 'application/xml', +} + +MIME_TYPES_ARCHIVE = { + 'application/zip', +} + +MIME_TYPES_TEXT_DATA = { + 'text/csv', + 'text/plain', +} + +MIME_TYPES_IMAGE = { + 'image/bmp', + 'image/gif', + 'image/jpeg', # jpeg, jpg + 'image/png', + 'image/svg', + 'image/svg+xml', + 'image/tiff', + 'image/webp', # shall we allow it? + 'image/x-ms-bmp', +} + +MIME_TYPES_AUDIO = { + 'audio/mp4', + 'audio/mpeg', + 'audio/wav', + 'audio/webm', # weba +} + +MIME_TYPES_VIDEO = { + 'video/mp4', + 'video/mpeg', # mpg, mpeg + 'video/ogg', + 'video/quicktime', # mov + 'video/webm', # webm + 'video/x-msvideo', # avi +} + + class WhitelistedMimeType: """ Makes sure an uploaded file is in a whitelist of allowed mimetypes. @@ -141,52 +205,13 @@ class WhitelistedMimeType: """ whitelist: Collection[str] = { - # documents - 'application/msword', # doc - 'application/pdf', - 'application/rtf', - 'application/vnd.ms-excel', # xls - ('application/vnd.openxmlformats-officedocument.' - 'presentationml.presentation'), # pptx - ('application/vnd.openxmlformats-officedocument.' - 'spreadsheetml.sheet'), # xlsx - ('application/vnd.openxmlformats-officedocument.' - 'wordprocessingml.document'), # docx - - # xml - 'application/xml', - - # archives - 'application/zip', - - # text / data - 'text/csv', - 'text/plain', - - # images - 'image/bmp', - 'image/gif', - 'image/jpeg', # jpeg, jpg - 'image/png', - 'image/svg', - 'image/svg+xml', - 'image/tiff', - 'image/webp', # shall we allow it? - 'image/x-ms-bmp', - - # audio - 'audio/mp4', - 'audio/mpeg', - 'audio/wav', - 'audio/webm', # weba - - # video - 'video/mp4', - 'video/mpeg', # mpg, mpeg - 'video/ogg', - 'video/quicktime', # mov - 'video/webm', # webm - 'video/x-msvideo', # avi + *MIME_TYPES_DOCUMENT, + *MIME_TYPES_XML, + *MIME_TYPES_ARCHIVE, + *MIME_TYPES_TEXT_DATA, + *MIME_TYPES_IMAGE, + *MIME_TYPES_AUDIO, + *MIME_TYPES_VIDEO, } message = _('Files of this type are not supported.') diff --git a/src/onegov/landsgemeinde/forms/agenda.py b/src/onegov/landsgemeinde/forms/agenda.py index e663d99d88..76114e2cce 100644 --- a/src/onegov/landsgemeinde/forms/agenda.py +++ b/src/onegov/landsgemeinde/forms/agenda.py @@ -19,7 +19,7 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_ARCHIVE from onegov.form.validators import WhitelistedMimeType from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout @@ -80,7 +80,7 @@ class AgendaItemForm(NamedFileForm): label=_('Excerpt from the Memorial (PDF)'), fieldset=_('Memorial'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -226,7 +226,7 @@ class AgendaItemUploadForm(Form): label=_('Agenda Item ZIP'), fieldset=_('Import'), validators=[ - WhitelistedMimeType({'application/zip'}), + WhitelistedMimeType(MIME_TYPES_ARCHIVE), FileSizeLimit(100 * 1024 * 1024) ] ) diff --git a/src/onegov/landsgemeinde/forms/assembly.py b/src/onegov/landsgemeinde/forms/assembly.py index ca4375e0fe..7449d0dd91 100644 --- a/src/onegov/landsgemeinde/forms/assembly.py +++ b/src/onegov/landsgemeinde/forms/assembly.py @@ -7,7 +7,7 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_AUDIO, MIME_TYPES_ARCHIVE from onegov.form.validators import WhitelistedMimeType from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout @@ -83,7 +83,7 @@ class AssemblyForm(NamedFileForm): label=_('Memorial part 1 (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -92,7 +92,7 @@ class AssemblyForm(NamedFileForm): label=_('Memorial part 2 (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -101,7 +101,7 @@ class AssemblyForm(NamedFileForm): label=_('Supplement to the memorial (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -110,7 +110,7 @@ class AssemblyForm(NamedFileForm): label=_('Protocol (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -119,7 +119,7 @@ class AssemblyForm(NamedFileForm): label=_('Audio (MP3)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'audio/mpeg'}), + WhitelistedMimeType(MIME_TYPES_AUDIO), FileSizeLimit(600 * 1024 * 1024) ] ) @@ -128,7 +128,7 @@ class AssemblyForm(NamedFileForm): label=_('Memorial as audio for the visually impaired and blind'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/zip'}), + WhitelistedMimeType(MIME_TYPES_ARCHIVE), FileSizeLimit(600 * 1024 * 1024) ] ) diff --git a/src/onegov/org/forms/directory.py b/src/onegov/org/forms/directory.py index aa6c707814..f6cc53a24d 100644 --- a/src/onegov/org/forms/directory.py +++ b/src/onegov/org/forms/directory.py @@ -17,7 +17,7 @@ from onegov.form.fields import IconField, MultiCheckboxField from onegov.form.fields import UploadField from onegov.form.filters import as_float -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_ARCHIVE from onegov.form.validators import ValidFormDefinition from onegov.form.validators import WhitelistedMimeType from onegov.org import _ diff --git a/src/onegov/org/forms/event.py b/src/onegov/org/forms/event.py index 4f571e1c00..888c50e681 100644 --- a/src/onegov/org/forms/event.py +++ b/src/onegov/org/forms/event.py @@ -537,19 +537,7 @@ class EventImportForm(Form): label=_('Import'), validators=[ DataRequired(), - WhitelistedMimeType({ - 'application/excel', - 'application/vnd.ms-excel', - ( - 'application/' - 'vnd.openxmlformats-officedocument.spreadsheetml.sheet' - ), - 'application/vnd.ms-office', - 'application/octet-stream', - 'application/zip', - 'text/csv', - 'text/plain', - }), + WhitelistedMimeType(), FileSizeLimit(10 * 1024 * 1024) ], render_kw={'force_simple': True} diff --git a/src/onegov/org/forms/parliamentarian.py b/src/onegov/org/forms/parliamentarian.py index b9ccf61ab5..f0bf28d7ae 100644 --- a/src/onegov/org/forms/parliamentarian.py +++ b/src/onegov/org/forms/parliamentarian.py @@ -6,7 +6,7 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ValidPhoneNumber, WhitelistedMimeType, MIME_TYPES_IMAGE from onegov.org import _ from onegov.parliament.models.parliamentarian import GENDERS from wtforms.fields import DateField @@ -49,6 +49,9 @@ class ParliamentarianForm(NamedFileForm): picture = UploadField( label=_('Picture'), fieldset=_('Basic properties'), + validators=[ + WhitelistedMimeType(MIME_TYPES_IMAGE) + ] ) party = StringField( diff --git a/src/onegov/pas/forms/data_import.py b/src/onegov/pas/forms/data_import.py index e5de118dcd..8394ce54f2 100644 --- a/src/onegov/pas/forms/data_import.py +++ b/src/onegov/pas/forms/data_import.py @@ -4,6 +4,7 @@ from onegov.core.utils import dictionary_to_binary from onegov.form import Form from onegov.form.fields import UploadMultipleField +from onegov.form.validators import WhitelistedMimeType, MIME_TYPES_JSON from onegov.pas import _ from onegov.pas.importer.json_import import ( MembershipData, @@ -24,6 +25,7 @@ class DataImportForm(Form): people_source = UploadMultipleField( label=_('People Data (JSON)'), description=_('JSON file containing parliamentarian data.'), + validators=[WhitelistedMimeType(MIME_TYPES_JSON)] ) organizations_source = UploadMultipleField( label=_('Organizations Data (JSON)'), @@ -31,6 +33,7 @@ class DataImportForm(Form): 'JSON file containing organization data (commissions, ' 'parties, etc.).' ), + validators=[WhitelistedMimeType(MIME_TYPES_JSON)] ) memberships_source = UploadMultipleField( label=_('Memberships Data (JSON)'), @@ -38,6 +41,7 @@ class DataImportForm(Form): 'JSON file containing membership data (who is member of ' 'what organization).' ), + validators=[WhitelistedMimeType(MIME_TYPES_JSON)] ) validate_schema = BooleanField( diff --git a/src/onegov/pas/forms/parliamentarian.py b/src/onegov/pas/forms/parliamentarian.py index 0f184186bc..5a87ada37b 100644 --- a/src/onegov/pas/forms/parliamentarian.py +++ b/src/onegov/pas/forms/parliamentarian.py @@ -4,7 +4,7 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ValidPhoneNumber, MIME_TYPES_IMAGE, WhitelistedMimeType from onegov.parliament.models.parliamentarian import GENDERS from onegov.parliament.models.parliamentarian import SHIPPING_METHODS from onegov.pas.collections.parliamentarian import ( @@ -65,6 +65,9 @@ class PASParliamentarianForm(NamedFileForm): picture = UploadField( label=_('Picture'), fieldset=_('Basic properties'), + validators=[ + WhitelistedMimeType(MIME_TYPES_IMAGE), + ] ) shipping_method = TranslatedSelectField( diff --git a/src/onegov/translator_directory/forms/accreditation.py b/src/onegov/translator_directory/forms/accreditation.py index c54c2500ea..0c07f83c06 100644 --- a/src/onegov/translator_directory/forms/accreditation.py +++ b/src/onegov/translator_directory/forms/accreditation.py @@ -14,7 +14,7 @@ from onegov.form.fields import PanelField from onegov.form.fields import TagsField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import Stdnum from onegov.form.validators import StrictOptional from onegov.form.validators import ValidPhoneNumber @@ -356,7 +356,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): declaration_of_authorization = UploadField( label=_('Signed declaration of authorization (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -367,7 +367,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): letter_of_motivation = UploadField( label=_('Short letter of motivation (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -378,7 +378,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): resume = UploadField( label=_('Resume (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -393,7 +393,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): 'level C2 are mandatory for non-native speakers.' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -404,7 +404,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): social_security_card = UploadField( label=_('Social security card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -415,7 +415,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): passport = UploadField( label=_('Identity card, passport or foreigner identity card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -426,7 +426,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): passport_photo = UploadField( label=_('Current passport photo (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -438,7 +438,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): label=_('Current extract from the debt collection register (PDF)'), description=_('Maximum 6 months since issue.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -453,7 +453,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): 'www.strafregister.admin.ch' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -465,7 +465,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): label=_('Certificate of Capability (PDF)'), description=_('Available from the municipal or city administration.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -479,7 +479,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): 'self-employment' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], diff --git a/src/onegov/translator_directory/forms/mutation.py b/src/onegov/translator_directory/forms/mutation.py index f32560861b..052ae20b39 100644 --- a/src/onegov/translator_directory/forms/mutation.py +++ b/src/onegov/translator_directory/forms/mutation.py @@ -12,7 +12,7 @@ from onegov.form.fields import MultiCheckboxField from onegov.form.fields import TagsField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import Stdnum from onegov.form.validators import ValidPhoneNumber from onegov.form.validators import ValidSwissSocialSecurityNumber @@ -256,7 +256,7 @@ def as_file(field: UploadField, category: str) -> File | None: declaration_of_authorization = UploadField( label=_('Signed declaration of authorization (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -267,7 +267,7 @@ def as_file(field: UploadField, category: str) -> File | None: letter_of_motivation = UploadField( label=_('Short letter of motivation (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -278,7 +278,7 @@ def as_file(field: UploadField, category: str) -> File | None: resume = UploadField( label=_('Resume (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -293,7 +293,7 @@ def as_file(field: UploadField, category: str) -> File | None: 'level C2 are mandatory for non-native speakers.' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -304,7 +304,7 @@ def as_file(field: UploadField, category: str) -> File | None: social_security_card = UploadField( label=_('Social security card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -315,7 +315,7 @@ def as_file(field: UploadField, category: str) -> File | None: passport = UploadField( label=_('Identity card, passport or foreigner identity card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -326,7 +326,7 @@ def as_file(field: UploadField, category: str) -> File | None: passport_photo = UploadField( label=_('Current passport photo (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -338,7 +338,7 @@ def as_file(field: UploadField, category: str) -> File | None: label=_('Current extract from the debt collection register (PDF)'), description=_('Maximum 6 months since issue.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -353,7 +353,7 @@ def as_file(field: UploadField, category: str) -> File | None: 'www.strafregister.admin.ch' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -365,7 +365,7 @@ def as_file(field: UploadField, category: str) -> File | None: label=_('Certificate of Capability (PDF)'), description=_('Available from the municipal or city administration.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -379,7 +379,7 @@ def as_file(field: UploadField, category: str) -> File | None: 'self-employment' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], From 1293c8e0cb52dfd26911c2c5fda4fe590b9719ce Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 11:57:55 +0100 Subject: [PATCH 10/36] Fix missing validator --- src/onegov/form/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 32c4a454bb..b8731d6ea3 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -485,6 +485,7 @@ def __init__( description=description, widget=upload_widget, render_kw=render_kw, + validators=validators, **extra_arguments ) super().__init__( From af685f5e5802eeaf00509869b8adb242c45434b6 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 11:58:53 +0100 Subject: [PATCH 11/36] Cleanup unused import --- src/onegov/org/forms/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/onegov/org/forms/directory.py b/src/onegov/org/forms/directory.py index f6cc53a24d..aa6c707814 100644 --- a/src/onegov/org/forms/directory.py +++ b/src/onegov/org/forms/directory.py @@ -17,7 +17,7 @@ from onegov.form.fields import IconField, MultiCheckboxField from onegov.form.fields import UploadField from onegov.form.filters import as_float -from onegov.form.validators import FileSizeLimit, MIME_TYPES_ARCHIVE +from onegov.form.validators import FileSizeLimit from onegov.form.validators import ValidFormDefinition from onegov.form.validators import WhitelistedMimeType from onegov.org import _ From a50f6f0e3c44ddab3e7222bd708213fa7a667104 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:01:50 +0100 Subject: [PATCH 12/36] Add fixme --- src/onegov/pas/views/data_import.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/onegov/pas/views/data_import.py b/src/onegov/pas/views/data_import.py index 25b16ee0f9..0445ddcd4e 100644 --- a/src/onegov/pas/views/data_import.py +++ b/src/onegov/pas/views/data_import.py @@ -95,6 +95,7 @@ def handle_data_import( error_message = None + # FIXME: why not use `form.submitted(request)` like in many places? if request.method == 'POST' and form.validate(): try: # Load and concatenate data from uploaded files From 458a82d85ce9aa9943c6a9ea497e652174fb004f Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:12:06 +0100 Subject: [PATCH 13/36] Fix linting errors --- src/onegov/form/fields.py | 2 +- src/onegov/landsgemeinde/forms/agenda.py | 8 ++++++-- src/onegov/landsgemeinde/forms/assembly.py | 9 +++++++-- src/onegov/org/forms/parliamentarian.py | 6 +++++- src/onegov/pas/forms/parliamentarian.py | 6 +++++- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index b8731d6ea3..89b6821803 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -485,7 +485,7 @@ def __init__( description=description, widget=upload_widget, render_kw=render_kw, - validators=validators, + validators=validators, # type:ignore[arg-type] **extra_arguments ) super().__init__( diff --git a/src/onegov/landsgemeinde/forms/agenda.py b/src/onegov/landsgemeinde/forms/agenda.py index 76114e2cce..07f590355c 100644 --- a/src/onegov/landsgemeinde/forms/agenda.py +++ b/src/onegov/landsgemeinde/forms/agenda.py @@ -19,8 +19,12 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_ARCHIVE -from onegov.form.validators import WhitelistedMimeType +from onegov.form.validators import ( + FileSizeLimit, + MIME_TYPES_PDF, + MIME_TYPES_ARCHIVE, + WhitelistedMimeType +) from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout from onegov.landsgemeinde.models import AgendaItem, LandsgemeindeFile diff --git a/src/onegov/landsgemeinde/forms/assembly.py b/src/onegov/landsgemeinde/forms/assembly.py index 7449d0dd91..e69affc5bc 100644 --- a/src/onegov/landsgemeinde/forms/assembly.py +++ b/src/onegov/landsgemeinde/forms/assembly.py @@ -7,8 +7,13 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_AUDIO, MIME_TYPES_ARCHIVE -from onegov.form.validators import WhitelistedMimeType +from onegov.form.validators import ( + FileSizeLimit, + MIME_TYPES_PDF, + MIME_TYPES_AUDIO, + MIME_TYPES_ARCHIVE, + WhitelistedMimeType +) from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout from onegov.landsgemeinde.models import Assembly, LandsgemeindeFile diff --git a/src/onegov/org/forms/parliamentarian.py b/src/onegov/org/forms/parliamentarian.py index f0bf28d7ae..39c44867ef 100644 --- a/src/onegov/org/forms/parliamentarian.py +++ b/src/onegov/org/forms/parliamentarian.py @@ -6,7 +6,11 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber, WhitelistedMimeType, MIME_TYPES_IMAGE +from onegov.form.validators import ( + MIME_TYPES_IMAGE, + ValidPhoneNumber, + WhitelistedMimeType +) from onegov.org import _ from onegov.parliament.models.parliamentarian import GENDERS from wtforms.fields import DateField diff --git a/src/onegov/pas/forms/parliamentarian.py b/src/onegov/pas/forms/parliamentarian.py index 5a87ada37b..4d0a85563f 100644 --- a/src/onegov/pas/forms/parliamentarian.py +++ b/src/onegov/pas/forms/parliamentarian.py @@ -4,7 +4,11 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber, MIME_TYPES_IMAGE, WhitelistedMimeType +from onegov.form.validators import ( + ValidPhoneNumber, + MIME_TYPES_IMAGE, + WhitelistedMimeType +) from onegov.parliament.models.parliamentarian import GENDERS from onegov.parliament.models.parliamentarian import SHIPPING_METHODS from onegov.pas.collections.parliamentarian import ( From 8d99429ee851494ef02c5d6e0de2640cd5e347bc Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:39:15 +0100 Subject: [PATCH 14/36] Extend test --- tests/onegov/form/test_fields.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 2b37849a9e..afe35b5dbf 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -279,12 +279,26 @@ def create_field( def test_upload_multiple_field() -> None: - def create_field() -> tuple[Form, UploadMultipleField]: + def create_field( + validators: Validators[FormT, Self] | None = None + ) -> tuple[Form, UploadMultipleField]: form = Form() - field = UploadMultipleField() + field = UploadMultipleField(validators=validators) field = field.bind(form, 'uploads') # type: ignore[attr-defined] return form, field + # failing mime type validator + form, field = create_field(validators=[ExpectedExtensions(['.json'])]) + file1 = create_file('text/plain', 'baz.txt', b'baz') + field.process(DummyPostData({'uploads': [file1]})) + assert not field.validate(form) + assert len(field.data) == 1 + assert field.data[0]['filename'] == 'baz.txt' + assert field.data[0]['mimetype'] == 'text/plain' + validator = find_validator(field, WhitelistedMimeType) + assert validator + assert validator.whitelist == {'application/json'} # type:ignore[attr-defined] + # Test rendering and initial submit form, field = create_field() field.process(None) @@ -321,8 +335,6 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert dictionary_to_binary(file_field2.data) == b'foobar' # type: ignore[arg-type] assert file_field2.filename == 'foobar.txt' assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] - - # verify attached validators assert_whitelisted_mimetype_validator(field) html = field(force_simple=True) From c31d0e3d5abb2796241bf3a548392b9ae428dd1e Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:57:25 +0100 Subject: [PATCH 15/36] Fix more linter issues --- tests/onegov/form/test_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index afe35b5dbf..4758c102c5 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -79,7 +79,7 @@ def create_file( def test_upload_field() -> None: def create_field( - validators: Validators[FormT, Self] | None = None + validators: Validators[FormT, Self] | None = None # type:ignore[misc] ) -> tuple[Form, UploadField]: form = Form() field = UploadField(validators=validators) @@ -280,7 +280,7 @@ def create_field( def test_upload_multiple_field() -> None: def create_field( - validators: Validators[FormT, Self] | None = None + validators: Validators[FormT, Self] | None = None # type:ignore[misc] ) -> tuple[Form, UploadMultipleField]: form = Form() field = UploadMultipleField(validators=validators) From eee980c1376789db10e7a4f6e7aa35036028be28 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 10:00:45 +0100 Subject: [PATCH 16/36] Remove validators from field list --- src/onegov/form/fields.py | 1 - tests/onegov/form/test_fields.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 89b6821803..db15643a94 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -498,7 +498,6 @@ def __init__( widget=widget, # type:ignore[arg-type] render_kw=render_kw, name=name, - validators=validators, _form=_form, _prefix=_prefix, _translations=_translations, diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 4758c102c5..fff4a52342 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -335,7 +335,8 @@ def create_field( assert dictionary_to_binary(file_field2.data) == b'foobar' # type: ignore[arg-type] assert file_field2.filename == 'foobar.txt' assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] - assert_whitelisted_mimetype_validator(field) + for subfield in field: + assert_whitelisted_mimetype_validator(subfield) html = field(force_simple=True) assert field.validate(form) From 248819adefac1ce0fb6a8752bb56c34aa7691c07 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 10:31:43 +0100 Subject: [PATCH 17/36] Add old ms office doc types --- src/onegov/form/validators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 5056747e44..d8d4a923c0 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -153,6 +153,9 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: 'spreadsheetml.sheet'), # xlsx ('application/vnd.openxmlformats-officedocument.' 'wordprocessingml.document'), # docx + 'application/CDFV2', # old ms office docs + 'application/x-ole-storage', # old ms office docs + 'application/CDFV2-unknown' # old ms office docs } MIME_TYPES_XML = { From 5621994564090923622218ce64ea7493440ef7a6 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 10:32:05 +0100 Subject: [PATCH 18/36] Remove non-standard svg type --- src/onegov/form/validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index d8d4a923c0..79b61eb2e9 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -176,7 +176,6 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: 'image/gif', 'image/jpeg', # jpeg, jpg 'image/png', - 'image/svg', 'image/svg+xml', 'image/tiff', 'image/webp', # shall we allow it? From 21274d1e9bd22fec8c59d2407d22302447efbf65 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 11:04:38 +0100 Subject: [PATCH 19/36] Update supported image mime type --- src/onegov/file/utils.py | 9 +-------- src/onegov/form/validators.py | 11 ++++------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/onegov/file/utils.py b/src/onegov/file/utils.py index f9daa6a386..75a8ce1549 100644 --- a/src/onegov/file/utils.py +++ b/src/onegov/file/utils.py @@ -74,15 +74,8 @@ def get_supported_image_mime_types() -> set[str]: # Not all PIL formats register a mime type, fill in the blanks ourselves. supported_types = { - 'image/bmp', - 'image/x-bmp', 'image/x-MS-bmp', - 'image/x-icon', - 'image/x-ico', - 'image/x-win-bitmap', - 'image/x-pcx', - 'image/x-portable-pixmap', - 'image/x-tga' + 'image/x-xcf', } for mime in Image.MIME.values(): diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 79b61eb2e9..d984dc618c 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -12,6 +12,8 @@ from decimal import Decimal from dateutil.relativedelta import relativedelta from mimetypes import types_map + +from onegov.file.utils import get_supported_image_mime_types from onegov.form import _ from onegov.form.errors import (DuplicateLabelError, InvalidIndentSyntax, EmptyFieldsetError) @@ -172,14 +174,9 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: } MIME_TYPES_IMAGE = { - 'image/bmp', - 'image/gif', - 'image/jpeg', # jpeg, jpg - 'image/png', + # allowed types based on PIL + *get_supported_image_mime_types(), 'image/svg+xml', - 'image/tiff', - 'image/webp', # shall we allow it? - 'image/x-ms-bmp', } MIME_TYPES_AUDIO = { From c4a1eefc2f7115dc6b4010126dbb30e535cd8040 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 11:16:03 +0100 Subject: [PATCH 20/36] Remove unused json validator --- src/onegov/pas/forms/data_import.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/onegov/pas/forms/data_import.py b/src/onegov/pas/forms/data_import.py index 8394ce54f2..1728f3ebcb 100644 --- a/src/onegov/pas/forms/data_import.py +++ b/src/onegov/pas/forms/data_import.py @@ -4,7 +4,6 @@ from onegov.core.utils import dictionary_to_binary from onegov.form import Form from onegov.form.fields import UploadMultipleField -from onegov.form.validators import WhitelistedMimeType, MIME_TYPES_JSON from onegov.pas import _ from onegov.pas.importer.json_import import ( MembershipData, @@ -25,7 +24,7 @@ class DataImportForm(Form): people_source = UploadMultipleField( label=_('People Data (JSON)'), description=_('JSON file containing parliamentarian data.'), - validators=[WhitelistedMimeType(MIME_TYPES_JSON)] + validators=[] # no validators as files are not stored ) organizations_source = UploadMultipleField( label=_('Organizations Data (JSON)'), @@ -33,7 +32,7 @@ class DataImportForm(Form): 'JSON file containing organization data (commissions, ' 'parties, etc.).' ), - validators=[WhitelistedMimeType(MIME_TYPES_JSON)] + validators=[] # no validators as files are not stored ) memberships_source = UploadMultipleField( label=_('Memberships Data (JSON)'), @@ -41,7 +40,7 @@ class DataImportForm(Form): 'JSON file containing membership data (who is member of ' 'what organization).' ), - validators=[WhitelistedMimeType(MIME_TYPES_JSON)] + validators=[] # no validators as files are not stored ) validate_schema = BooleanField( From e77bc498c334f1baaba317a32a678357e7e34443 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 8 Dec 2025 08:06:40 +0100 Subject: [PATCH 21/36] Pass validators only to FieldList and introduce allowed mime types --- src/onegov/form/fields.py | 80 ++++++++++++++++++++--------- src/onegov/pas/forms/data_import.py | 10 ++-- tests/onegov/form/test_fields.py | 31 +++++++---- 3 files changed, 82 insertions(+), 39 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index db15643a94..26f5b367b4 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -260,30 +260,59 @@ class UploadField(FileField): action: Literal['keep', 'replace', 'delete'] file: IO[bytes] | None filename: str | None - validators = [WhitelistedMimeType()] - if TYPE_CHECKING: - def __init__( - self, - label: str | None = None, - validators: Validators[FormT, Self] | None = None, - filters: Sequence[Filter] = (), - description: str = '', - id: str | None = None, - default: Sequence[StrictFileDict] = (), - widget: Widget[Self] | None = None, - render_kw: dict[str, Any] | None = None, - name: str | None = None, - _form: BaseForm | None = None, - _prefix: str = '', - _translations: _SupportsGettextAndNgettext | None = None, - _meta: DefaultMeta | None = None, - # onegov specific kwargs that get popped off - *, - fieldset: str | None = None, - depends_on: Sequence[Any] | None = None, - pricing: PricingRules | None = None, - ): ... + def __init__( + self, + label: str | None = None, + validators: Validators[FormT, Self] | None = None, + filters: Sequence[Filter] = (), + description: str = '', + id: str | None = None, + default: Sequence[StrictFileDict] = (), + widget: Widget[Self] | None = None, + render_kw: dict[str, Any] | None = None, + name: str | None = None, + allowed_mimetypes: Sequence[str] | None = None, + _form: BaseForm | None = None, + _prefix: str = '', + _translations: _SupportsGettextAndNgettext | None = None, + _meta: DefaultMeta | None = None, + # onegov specific kwargs that get popped off + *, + fieldset: str | None = None, + depends_on: Sequence[Any] | None = None, + pricing: PricingRules | None = None, + ): + validator = ( + WhitelistedMimeType(allowed_mimetypes) + if allowed_mimetypes + else WhitelistedMimeType() + ) + + if validators: + validators = list(validators) + if not any(isinstance(validator, WhitelistedMimeType) + for validator in validators + ): + validators.append(validator) + else: + validators = [validator] + + super().__init__( + label=label, + validators=validators, + filters=filters, + description=description, + id=id, + default=default, + widget=widget, + render_kw=render_kw, + name=name, + _form=_form, + _prefix=_prefix, + _translations=_translations, + _meta=_meta, + ) # this is not quite accurate, since it is either a dictionary with all # the keys or none of the keys, which would make type narrowing easier @@ -449,7 +478,6 @@ def _add_entry(self, d: _MultiDictLikeWithGetlist, /) -> UploadField: upload_field_class: type[UploadField] = UploadField upload_widget: Widget[UploadField] = UploadWidget() - validators = [WhitelistedMimeType()] def __init__( self, @@ -463,6 +491,7 @@ def __init__( render_kw: dict[str, Any] | None = None, name: str | None = None, upload_widget: Widget[UploadField] | None = None, + allowed_mimetypes: Sequence[str] | None = None, _form: BaseForm | None = None, _prefix: str = '', _translations: _SupportsGettextAndNgettext | None = None, @@ -485,7 +514,7 @@ def __init__( description=description, widget=upload_widget, render_kw=render_kw, - validators=validators, # type:ignore[arg-type] + allowed_mimetypes=allowed_mimetypes, **extra_arguments ) super().__init__( @@ -496,6 +525,7 @@ def __init__( id=id, default=default, widget=widget, # type:ignore[arg-type] + validators=[*(validators or ())], render_kw=render_kw, name=name, _form=_form, diff --git a/src/onegov/pas/forms/data_import.py b/src/onegov/pas/forms/data_import.py index 1728f3ebcb..1ec6b24926 100644 --- a/src/onegov/pas/forms/data_import.py +++ b/src/onegov/pas/forms/data_import.py @@ -4,6 +4,7 @@ from onegov.core.utils import dictionary_to_binary from onegov.form import Form from onegov.form.fields import UploadMultipleField +from onegov.form.validators import MIME_TYPES_JSON from onegov.pas import _ from onegov.pas.importer.json_import import ( MembershipData, @@ -24,7 +25,8 @@ class DataImportForm(Form): people_source = UploadMultipleField( label=_('People Data (JSON)'), description=_('JSON file containing parliamentarian data.'), - validators=[] # no validators as files are not stored + validators=[], # no validators as files are not stored + allowed_mimetypes=tuple(MIME_TYPES_JSON), ) organizations_source = UploadMultipleField( label=_('Organizations Data (JSON)'), @@ -32,7 +34,8 @@ class DataImportForm(Form): 'JSON file containing organization data (commissions, ' 'parties, etc.).' ), - validators=[] # no validators as files are not stored + validators=[], # no validators as files are not stored + allowed_mimetypes=tuple(MIME_TYPES_JSON), ) memberships_source = UploadMultipleField( label=_('Memberships Data (JSON)'), @@ -40,7 +43,8 @@ class DataImportForm(Form): 'JSON file containing membership data (who is member of ' 'what organization).' ), - validators=[] # no validators as files are not stored + validators=[], # no validators as files are not stored + allowed_mimetypes=tuple(MIME_TYPES_JSON), ) validate_schema = BooleanField( diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index fff4a52342..2f0d9c5aab 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -24,14 +24,14 @@ from onegov.form.fields import UploadMultipleField from onegov.form.fields import URLField from onegov.form.validators import ( - ValidPhoneNumber, WhitelistedMimeType, ExpectedExtensions) + ValidPhoneNumber, WhitelistedMimeType, MIME_TYPES_JSON, MIME_TYPES_PDF) from unittest.mock import patch from wtforms import FileField, Field from wtforms.validators import Optional from wtforms.validators import URL -from typing import Any, TYPE_CHECKING, Self +from typing import Any, TYPE_CHECKING, Self, Sequence if TYPE_CHECKING: from webob.request import _FieldStorageWithFile @@ -79,10 +79,14 @@ def create_file( def test_upload_field() -> None: def create_field( - validators: Validators[FormT, Self] | None = None # type:ignore[misc] + validators: Validators[FormT, Self] | None = None, # type:ignore[misc] + allowed_mimetypes: Sequence[str] | None = None, ) -> tuple[Form, UploadField]: form = Form() - field = UploadField(validators=validators) + field = UploadField( + validators=validators, + allowed_mimetypes=allowed_mimetypes, + ) field = field.bind(form, 'upload') # type: ignore[attr-defined] return form, field @@ -128,7 +132,7 @@ def create_field( assert_whitelisted_mimetype_validator(field) # failing mime type validator - form, field = create_field(validators=[ExpectedExtensions(['.pdf'])]) + form, field = create_field(allowed_mimetypes=tuple(MIME_TYPES_PDF)) textfile = create_file('text/plain', 'baz.txt', b'baz') data = field.data = field.process_fieldstorage(textfile) assert data['filename'] == 'baz.txt' @@ -280,24 +284,29 @@ def create_field( def test_upload_multiple_field() -> None: def create_field( - validators: Validators[FormT, Self] | None = None # type:ignore[misc] + validators: Validators[FormT, Self] | None = None, # type:ignore[misc] + allowed_mimetypes: Sequence[str] | None = None, ) -> tuple[Form, UploadMultipleField]: form = Form() - field = UploadMultipleField(validators=validators) + field = UploadMultipleField( + validators=validators, + allowed_mimetypes=allowed_mimetypes, + ) field = field.bind(form, 'uploads') # type: ignore[attr-defined] return form, field # failing mime type validator - form, field = create_field(validators=[ExpectedExtensions(['.json'])]) + form, field = create_field(allowed_mimetypes=tuple(MIME_TYPES_JSON)) file1 = create_file('text/plain', 'baz.txt', b'baz') field.process(DummyPostData({'uploads': [file1]})) assert not field.validate(form) assert len(field.data) == 1 assert field.data[0]['filename'] == 'baz.txt' assert field.data[0]['mimetype'] == 'text/plain' - validator = find_validator(field, WhitelistedMimeType) - assert validator - assert validator.whitelist == {'application/json'} # type:ignore[attr-defined] + for subfield in field: + validator = find_validator(subfield, WhitelistedMimeType) + assert validator + assert validator.whitelist == {'application/json'} # type:ignore[attr-defined] # Test rendering and initial submit form, field = create_field() From 9a6a18700c45f8fc1063b294f57d8caf193cca18 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 8 Dec 2025 08:27:38 +0100 Subject: [PATCH 22/36] Adjust tests --- tests/onegov/form/test_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 2f0d9c5aab..0f1f15b9fc 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -139,7 +139,7 @@ def create_field( assert data['mimetype'] == 'text/plain' validator = find_validator(field, WhitelistedMimeType) assert validator - assert validator.whitelist == {'application/pdf'} # type:ignore[attr-defined] + assert validator.whitelist == ('application/pdf',) # type:ignore[attr-defined] assert not field.validate(form) assert 'Files of this type are not supported.' in field.errors @@ -306,7 +306,7 @@ def create_field( for subfield in field: validator = find_validator(subfield, WhitelistedMimeType) assert validator - assert validator.whitelist == {'application/json'} # type:ignore[attr-defined] + assert validator.whitelist == ('application/json',) # type:ignore[attr-defined] # Test rendering and initial submit form, field = create_field() From d122951692763ec4c38779c8f7bb99c5ed2b5bc0 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 8 Dec 2025 10:53:30 +0100 Subject: [PATCH 23/36] Fix wrong default value for UploadField --- src/onegov/form/fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 26f5b367b4..444ba8b530 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -59,7 +59,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterator, Sequence from datetime import datetime - from onegov.core.types import FileDict as StrictFileDict + from onegov.core.types import FileDict as StrictFileDict, LaxFileDict from onegov.file import File from onegov.form import Form from onegov.form.types import ( @@ -268,7 +268,7 @@ def __init__( filters: Sequence[Filter] = (), description: str = '', id: str | None = None, - default: Sequence[StrictFileDict] = (), + default: LaxFileDict | None = None, widget: Widget[Self] | None = None, render_kw: dict[str, Any] | None = None, name: str | None = None, From 1e7b28b2fdc388d0877495c3b4efe324db4c968b Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 8 Dec 2025 11:06:42 +0100 Subject: [PATCH 24/36] Fix linter issues --- tests/onegov/form/test_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 0f1f15b9fc..2f0d9c5aab 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -139,7 +139,7 @@ def create_field( assert data['mimetype'] == 'text/plain' validator = find_validator(field, WhitelistedMimeType) assert validator - assert validator.whitelist == ('application/pdf',) # type:ignore[attr-defined] + assert validator.whitelist == {'application/pdf'} # type:ignore[attr-defined] assert not field.validate(form) assert 'Files of this type are not supported.' in field.errors @@ -306,7 +306,7 @@ def create_field( for subfield in field: validator = find_validator(subfield, WhitelistedMimeType) assert validator - assert validator.whitelist == ('application/json',) # type:ignore[attr-defined] + assert validator.whitelist == {'application/json'} # type:ignore[attr-defined] # Test rendering and initial submit form, field = create_field() From 427563eadd357f0c2e0400f0f28a2585c2f3e28c Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 8 Dec 2025 11:43:20 +0100 Subject: [PATCH 25/36] Limit event import to excel kind of files --- src/onegov/form/validators.py | 11 +++++++++++ src/onegov/org/forms/event.py | 8 ++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index d984dc618c..96f0aa4454 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -148,6 +148,7 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: 'application/msword', # doc 'application/rtf', *MIME_TYPES_PDF, + 'application/excel', 'application/vnd.ms-excel', # xls ('application/vnd.openxmlformats-officedocument.' 'presentationml.presentation'), # pptx @@ -155,11 +156,21 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: 'spreadsheetml.sheet'), # xlsx ('application/vnd.openxmlformats-officedocument.' 'wordprocessingml.document'), # docx + 'application/vnd.ms-office', 'application/CDFV2', # old ms office docs 'application/x-ole-storage', # old ms office docs 'application/CDFV2-unknown' # old ms office docs } +MIME_TYPES_EXCEL = { + 'application/excel', + 'application/vnd.ms-excel,' + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.ms-office', + 'application/octet-stream', + 'application/x-ole-storage', +} + MIME_TYPES_XML = { 'application/xml', } diff --git a/src/onegov/org/forms/event.py b/src/onegov/org/forms/event.py index 888c50e681..9f8d68dd0f 100644 --- a/src/onegov/org/forms/event.py +++ b/src/onegov/org/forms/event.py @@ -21,7 +21,11 @@ from onegov.form.fields import UploadFileWithORMSupport from onegov.form.utils import get_fields_from_class from onegov.form.validators import ( - FileSizeLimit, ValidPhoneNumber, ValidFilterFormDefinition) + FileSizeLimit, + ValidPhoneNumber, + ValidFilterFormDefinition, + MIME_TYPES_EXCEL +) from onegov.form.validators import WhitelistedMimeType from onegov.gis import CoordinatesField from onegov.org import _ @@ -537,7 +541,7 @@ class EventImportForm(Form): label=_('Import'), validators=[ DataRequired(), - WhitelistedMimeType(), + WhitelistedMimeType(MIME_TYPES_EXCEL), FileSizeLimit(10 * 1024 * 1024) ], render_kw={'force_simple': True} From fa438e045da4a0912218f5954b4612fa5f62385d Mon Sep 17 00:00:00 2001 From: Reto Tschuppert <124258444+Tschuppi81@users.noreply.github.com> Date: Mon, 8 Dec 2025 07:50:47 -0500 Subject: [PATCH 26/36] Improve validator type Co-authored-by: David Salvisberg --- tests/onegov/form/test_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 2f0d9c5aab..027075d8ab 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -50,8 +50,8 @@ def assert_whitelisted_mimetype_validator( def find_validator( field: Field | FileField, - cls: type -) -> Validator[Any, Any] | None: + cls: type[ValidatorT] +) -> ValidatorT | None: return next((v for v in field.validators if isinstance(v, cls)), None) From 0f7b2105767b4982491c49e0dcce37d58a8cce47 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 15 Dec 2025 13:36:17 +0100 Subject: [PATCH 27/36] Verify file type for file collection upload --- src/onegov/org/assets/js/upload.js | 5 +++++ src/onegov/org/models/file.py | 3 ++- src/onegov/org/views/files.py | 15 +++++++++------ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/onegov/org/assets/js/upload.js b/src/onegov/org/assets/js/upload.js index 203c450408..56177ff037 100644 --- a/src/onegov/org/assets/js/upload.js +++ b/src/onegov/org/assets/js/upload.js @@ -41,6 +41,7 @@ var Upload = function(element) { var data = new FormData(); data.append('file', file); + data.append('file_type', file.type); xhr.upload.addEventListener('progress', function(e) { bar.find('.meter').css('width', (e.loaded / e.total * 100 || 100) + '%'); @@ -60,6 +61,10 @@ var Upload = function(element) { if (xhr.responseText.length !== 0) { processCommonNodes($(xhr.responseText).appendTo(filelist), true); } + } else if (xhr.status === 415){ // Unsupported media type + // reload page in order to show the request error message + window.location.reload(); + return; } else { bar.find('.meter').css('width', '100%'); bar.addClass('alert').attr('data-error', xhr.statusText); diff --git a/src/onegov/org/models/file.py b/src/onegov/org/models/file.py index 7588189a28..a8881c4f7e 100644 --- a/src/onegov/org/models/file.py +++ b/src/onegov/org/models/file.py @@ -12,6 +12,7 @@ from onegov.file import File, FileSet, FileCollection, FileSetCollection from onegov.file import SearchableFile from onegov.file.utils import IMAGE_MIME_TYPES_AND_SVG +from onegov.form.validators import WhitelistedMimeType from onegov.org import _ from onegov.org.models.extensions import AccessExtension from onegov.org.utils import widest_access @@ -311,7 +312,7 @@ class GeneralFileCollection( GroupFilesByDateMixin[GeneralFile] ): - supported_content_types = 'all' + supported_content_types = WhitelistedMimeType.whitelist file_list = as_selectable(""" SELECT diff --git a/src/onegov/org/views/files.py b/src/onegov/org/views/files.py index f8a4208f81..a7e48bb495 100644 --- a/src/onegov/org/views/files.py +++ b/src/onegov/org/views/files.py @@ -434,6 +434,14 @@ def handle_file_upload( """ + file_type = request.params['file_type'] + supported_content_types = getattr(self, 'supported_content_types') + + if supported_content_types != 'all': + if file_type not in supported_content_types: + request.alert(_('This file type is not supported')) + raise exc.HTTPUnsupportedMediaType() + fs = request.params['file'] assert not isinstance(fs, str) @@ -442,12 +450,6 @@ def handle_file_upload( content=fs.file ) - supported_content_types = getattr(self, 'supported_content_types', 'all') - - if supported_content_types != 'all': - if file.reference.content_type not in supported_content_types: - raise exc.HTTPUnsupportedMediaType() - return file @@ -474,6 +476,7 @@ def view_upload_file( request: OrgRequest, return_file: bool = False ) -> Response | File: + """ Gets called from upload.js Upload class """ request.assert_valid_csrf_token() From e5552d68ddb2538b4af87aed59c1e85369b7a0c1 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 15 Dec 2025 14:20:39 +0100 Subject: [PATCH 28/36] Adjust tests --- tests/onegov/form/test_fields.py | 8 ++++---- tests/onegov/form/test_parser.py | 24 +++++++++++++++++++----- tests/onegov/pas/test_views.py | 1 - 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 027075d8ab..0f1f15b9fc 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -50,8 +50,8 @@ def assert_whitelisted_mimetype_validator( def find_validator( field: Field | FileField, - cls: type[ValidatorT] -) -> ValidatorT | None: + cls: type +) -> Validator[Any, Any] | None: return next((v for v in field.validators if isinstance(v, cls)), None) @@ -139,7 +139,7 @@ def create_field( assert data['mimetype'] == 'text/plain' validator = find_validator(field, WhitelistedMimeType) assert validator - assert validator.whitelist == {'application/pdf'} # type:ignore[attr-defined] + assert validator.whitelist == ('application/pdf',) # type:ignore[attr-defined] assert not field.validate(form) assert 'Files of this type are not supported.' in field.errors @@ -306,7 +306,7 @@ def create_field( for subfield in field: validator = find_validator(subfield, WhitelistedMimeType) assert validator - assert validator.whitelist == {'application/json'} # type:ignore[attr-defined] + assert validator.whitelist == ('application/json',) # type:ignore[attr-defined] # Test rendering and initial submit form, field = create_field() diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index 6acea95e59..2278a57c40 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -18,8 +18,13 @@ VideoURLField, ) from onegov.form.parser.grammar import field_help_identifier -from onegov.form.validators import LaxDataRequired, WhitelistedMimeType -from onegov.form.validators import ValidDateRange +from onegov.form.validators import ( + LaxDataRequired, + WhitelistedMimeType, + FileSizeLimit, + StrictOptional, + ValidDateRange +) from onegov.pay import Price from textwrap import dedent from webob.multidict import MultiDict @@ -32,7 +37,6 @@ from wtforms.validators import Optional from wtforms.validators import Regexp - from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -360,6 +364,7 @@ def test_parse_fileinput() -> None: assert form['file'].widget.multiple is False # type: ignore[attr-defined] # verify attached mime type validator + assert find_validator(form['file'], FileSizeLimit) assert form['file'].validators validator = find_validator(form['file'], WhitelistedMimeType) assert validator @@ -372,6 +377,13 @@ def test_parse_fileinput() -> None: assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] + assert find_validator(form['file'], FileSizeLimit) + + # ensure nickname field did not get validators from the upload field + form = parse_form("Nickname = ___\nFile = *.pdf|*.doc")() + assert find_validator(form['nickname'], StrictOptional) + assert not find_validator(form['nickname'], WhitelistedMimeType) + def test_parse_multiplefileinput() -> None: form = parse_form("Files = *.pdf|*.doc (multiple)")() @@ -387,11 +399,13 @@ def test_parse_multiplefileinput() -> None: 'application/msword', 'application/pdf' } - form = parse_form("Files = *.* (multiple)")() - validator = find_validator(form['files'], WhitelistedMimeType) + form = parse_form("My files = *.* (multiple)")() + validator = find_validator(form['my_files'], WhitelistedMimeType) assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] + assert find_validator(form['my_files'], FileSizeLimit) + def test_parse_radio() -> None: text = dedent(""" diff --git a/tests/onegov/pas/test_views.py b/tests/onegov/pas/test_views.py index 151f7652b3..c6c0f02cd9 100644 --- a/tests/onegov/pas/test_views.py +++ b/tests/onegov/pas/test_views.py @@ -382,7 +382,6 @@ def do_upload_procedure( result = page.form.submit().maybe_follow() # Add assertions as needed - assert result.status_code == 200 assert result.status_code == 200, f"Import failed: {result.text}" return result From a5454774d5d026699250a72e65eb01312b767124 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 15 Dec 2025 14:21:05 +0100 Subject: [PATCH 29/36] Set default mime type white list for translator directory as well --- src/onegov/translator_directory/collections/documents.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/onegov/translator_directory/collections/documents.py b/src/onegov/translator_directory/collections/documents.py index 9fbcceba17..68ea1c7613 100644 --- a/src/onegov/translator_directory/collections/documents.py +++ b/src/onegov/translator_directory/collections/documents.py @@ -4,6 +4,7 @@ from itertools import groupby from onegov.file import File, FileCollection +from onegov.form.validators import WhitelistedMimeType from onegov.translator_directory.models.translator import Translator @@ -29,7 +30,7 @@ class TranslatorDocumentCollection(FileCollection[File]): - supported_content_types = 'all' + supported_content_types = WhitelistedMimeType.whitelist def __init__( self, From 5f7bcb263178c2d48f2349dcc9e4d8c71901b1b3 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 18 Dec 2025 09:40:03 +0100 Subject: [PATCH 30/36] Revert "Verify file type for file collection upload" This reverts commit 0f7b2105767b4982491c49e0dcce37d58a8cce47. --- src/onegov/org/assets/js/upload.js | 5 ----- src/onegov/org/models/file.py | 3 +-- src/onegov/org/views/files.py | 15 ++++++--------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/onegov/org/assets/js/upload.js b/src/onegov/org/assets/js/upload.js index 56177ff037..203c450408 100644 --- a/src/onegov/org/assets/js/upload.js +++ b/src/onegov/org/assets/js/upload.js @@ -41,7 +41,6 @@ var Upload = function(element) { var data = new FormData(); data.append('file', file); - data.append('file_type', file.type); xhr.upload.addEventListener('progress', function(e) { bar.find('.meter').css('width', (e.loaded / e.total * 100 || 100) + '%'); @@ -61,10 +60,6 @@ var Upload = function(element) { if (xhr.responseText.length !== 0) { processCommonNodes($(xhr.responseText).appendTo(filelist), true); } - } else if (xhr.status === 415){ // Unsupported media type - // reload page in order to show the request error message - window.location.reload(); - return; } else { bar.find('.meter').css('width', '100%'); bar.addClass('alert').attr('data-error', xhr.statusText); diff --git a/src/onegov/org/models/file.py b/src/onegov/org/models/file.py index a8881c4f7e..7588189a28 100644 --- a/src/onegov/org/models/file.py +++ b/src/onegov/org/models/file.py @@ -12,7 +12,6 @@ from onegov.file import File, FileSet, FileCollection, FileSetCollection from onegov.file import SearchableFile from onegov.file.utils import IMAGE_MIME_TYPES_AND_SVG -from onegov.form.validators import WhitelistedMimeType from onegov.org import _ from onegov.org.models.extensions import AccessExtension from onegov.org.utils import widest_access @@ -312,7 +311,7 @@ class GeneralFileCollection( GroupFilesByDateMixin[GeneralFile] ): - supported_content_types = WhitelistedMimeType.whitelist + supported_content_types = 'all' file_list = as_selectable(""" SELECT diff --git a/src/onegov/org/views/files.py b/src/onegov/org/views/files.py index a7e48bb495..f8a4208f81 100644 --- a/src/onegov/org/views/files.py +++ b/src/onegov/org/views/files.py @@ -434,14 +434,6 @@ def handle_file_upload( """ - file_type = request.params['file_type'] - supported_content_types = getattr(self, 'supported_content_types') - - if supported_content_types != 'all': - if file_type not in supported_content_types: - request.alert(_('This file type is not supported')) - raise exc.HTTPUnsupportedMediaType() - fs = request.params['file'] assert not isinstance(fs, str) @@ -450,6 +442,12 @@ def handle_file_upload( content=fs.file ) + supported_content_types = getattr(self, 'supported_content_types', 'all') + + if supported_content_types != 'all': + if file.reference.content_type not in supported_content_types: + raise exc.HTTPUnsupportedMediaType() + return file @@ -476,7 +474,6 @@ def view_upload_file( request: OrgRequest, return_file: bool = False ) -> Response | File: - """ Gets called from upload.js Upload class """ request.assert_valid_csrf_token() From 5c54c0629fc95c25c9dcf17fe97d82c09e295ff6 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 18 Dec 2025 15:18:11 +0100 Subject: [PATCH 31/36] Make unsupported media type error visible on files view --- src/onegov/org/assets/js/locale.js | 12 +++-- src/onegov/org/assets/js/upload.js | 25 +++++++-- src/onegov/org/models/file.py | 3 +- src/onegov/org/views/files.py | 7 ++- src/onegov/town6/theme/styles/upload.scss | 65 +++++++++++++++++------ 5 files changed, 86 insertions(+), 26 deletions(-) diff --git a/src/onegov/org/assets/js/locale.js b/src/onegov/org/assets/js/locale.js index 5771369e55..3a6226d9f0 100644 --- a/src/onegov/org/assets/js/locale.js +++ b/src/onegov/org/assets/js/locale.js @@ -30,7 +30,9 @@ var locales = { "The server responded with an error. We have been informed and will investigate the problem.": "Auf dem Server ist ein Fehler aufgetreten. Wir wurden informiert und werden das Problem analysieren.", "The server could not be reached. Please try again.": "Der Server konnte nicht erreicht werden. Bitte probieren Sie es noch einmal.", "The site could not be found.": "Die Seite wurde nicht gefunden.", - "Access denied. Please log in before continuing.": "Zugriff verweigert. Bitte melden Sie sich an bevor Sie weiterfahren." + "Access denied. Please log in before continuing.": "Zugriff verweigert. Bitte melden Sie sich an bevor Sie weiterfahren.", + "Unsupported Media Type": "Nicht unterstützter Medientyp", + "Upload failed": "Upload fehlgeschlagen" }, fr: { "Allocation": "Allocation", @@ -63,7 +65,9 @@ var locales = { "The server responded with an error. We have been informed and will investigate the problem.": "Le serveur a répondu par une erreur. Nous en avons été informés et nous étudierons le problème.", "The server could not be reached. Please try again.": "Le serveur n'a pas pu être joint. Veuillez réessayer.", "The site could not be found.": "Impossible de trouver le site.", - "Access denied. Please log in before continuing.": "Accès refusé. Veuillez vous connecter avant de continuer." + "Access denied. Please log in before continuing.": "Accès refusé. Veuillez vous connecter avant de continuer.", + "Unsupported Media Type": "Type de média non supporté", + "Upload failed": "L'envoi a échoué" }, it: { "Allocation": "Allocazione", @@ -96,7 +100,9 @@ var locales = { "The server responded with an error. We have been informed and will investigate the problem.": "Il server ha risposto con un errore. Siamo stati informati e indagheremo sul problema.", "The server could not be reached. Please try again.": "Impossibile raggiungere il server. Per favore riprova.", "The site could not be found.": "Impossibile trovare il sito.", - "Access denied. Please log in before continuing.": "Accesso negato. Effettua il login prima di continuare." + "Access denied. Please log in before continuing.": "Accesso negato. Effettua il login prima di continuare.", + "Unsupported Media Type": "Tipo di media non supportato", + "Upload failed": "L'upload non è andato a buon fine" } }; diff --git a/src/onegov/org/assets/js/upload.js b/src/onegov/org/assets/js/upload.js index 203c450408..905b3491c0 100644 --- a/src/onegov/org/assets/js/upload.js +++ b/src/onegov/org/assets/js/upload.js @@ -61,8 +61,15 @@ var Upload = function(element) { processCommonNodes($(xhr.responseText).appendTo(filelist), true); } } else { + var errText = locale(xhr.statusText || 'Upload failed') ; + bar.find('.meter').css('width', '100%'); - bar.addClass('alert').attr('data-error', xhr.statusText); + bar.addClass('has-error').attr('data-error', errText); + bar.find('.upload-error').text(errText).attr('aria-hidden', 'false').show(); + bar.attr('title', errText); + + // keep the bar visible but mark as failed + bar.find('.upload-filename').addClass('muted'); } // eslint-disable-next-line no-use-before-define @@ -82,9 +89,19 @@ var Upload = function(element) { }; var queue_upload = function(file) { - var bar = $('
') - .attr('data-filename', file.name) - .prependTo(progress); + var bar = $( + '
' + + '
' + + '' + + '' + + '
' + + '' + + '
' + ).attr('data-filename', file.name) + .prependTo(progress); + + bar.find('.upload-filename').text(file.name); + upload_queue.push({file: file, bar: bar}); }; diff --git a/src/onegov/org/models/file.py b/src/onegov/org/models/file.py index 7588189a28..a8881c4f7e 100644 --- a/src/onegov/org/models/file.py +++ b/src/onegov/org/models/file.py @@ -12,6 +12,7 @@ from onegov.file import File, FileSet, FileCollection, FileSetCollection from onegov.file import SearchableFile from onegov.file.utils import IMAGE_MIME_TYPES_AND_SVG +from onegov.form.validators import WhitelistedMimeType from onegov.org import _ from onegov.org.models.extensions import AccessExtension from onegov.org.utils import widest_access @@ -311,7 +312,7 @@ class GeneralFileCollection( GroupFilesByDateMixin[GeneralFile] ): - supported_content_types = 'all' + supported_content_types = WhitelistedMimeType.whitelist file_list = as_selectable(""" SELECT diff --git a/src/onegov/org/views/files.py b/src/onegov/org/views/files.py index f8a4208f81..caf2ed895e 100644 --- a/src/onegov/org/views/files.py +++ b/src/onegov/org/views/files.py @@ -442,10 +442,15 @@ def handle_file_upload( content=fs.file ) - supported_content_types = getattr(self, 'supported_content_types', 'all') + supported_content_types = self.supported_content_types # type:ignore[attr-defined] if supported_content_types != 'all': if file.reference.content_type not in supported_content_types: + # Fail the post request from upload.js with status code 415 + # (Unsupported Media Type). Raising the HTTP exception here causes + # the request transaction to abort and roll back any previous + # changes (including the `self.add(...)` above), so the file won't + # be persisted if the content type is unsupported. raise exc.HTTPUnsupportedMediaType() return file diff --git a/src/onegov/town6/theme/styles/upload.scss b/src/onegov/town6/theme/styles/upload.scss index 40d4fbcacf..5a13e694fc 100644 --- a/src/onegov/town6/theme/styles/upload.scss +++ b/src/onegov/town6/theme/styles/upload.scss @@ -39,27 +39,58 @@ margin-bottom: 1.5rem; .progress { - margin-left: 25%; - width: 75%; + padding: 0.5rem; + background: #f6f6f6; + position: relative; + overflow: visible; - &::before { - content: attr(data-filename); - font-size: .875rem; - left: 1rem; + .progress-main { + display: flex; + align-items: center; + gap: 0.75rem; + } + + .upload-filename { + flex: 0 0 auto; + font-size: 0.95rem; + color: #333; + max-width: 50%; + white-space: nowrap; overflow: hidden; - text-align: left; text-overflow: ellipsis; - white-space: nowrap; - width: 22.5%; } - &.alert::after { - color: $white; - content: attr(data-error); - font-size: .8rem; - margin-left: .75ex; - margin-top: -1.2rem; - position: absolute; + .meter { + display: inline-block; + height: 10px; + background: #0b7db1; + border-radius: 6px; + flex: 1 1 auto; + transition: width 150ms linear; + } + + .upload-error { + margin-top: 0.4rem; + color: #842029; + background: #f8d7da; + padding: 0.4rem 0.6rem; + font-size: 0.875rem; + display: none; + } + + &.has-error { + .meter { background: #b02a37; } + .upload-error { display: block; } + } + + // optional compact mobile layout + @media (max-width: 640px) { + .progress-main { + flex-direction: column; + align-items: stretch; + gap: 0.5rem; + } + .upload-filename { max-width: 100%; } } } -} +} \ No newline at end of file From f839024e586170f6e4a25b27b11ba6572fce2e6a Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 18 Dec 2025 15:19:11 +0100 Subject: [PATCH 32/36] Align upload columns with already uploaded files --- src/onegov/town6/templates/files.pt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/onegov/town6/templates/files.pt b/src/onegov/town6/templates/files.pt index 334d010c4c..cea086a882 100644 --- a/src/onegov/town6/templates/files.pt +++ b/src/onegov/town6/templates/files.pt @@ -24,6 +24,9 @@ Upload Date + + Published until + From 789e50399bb712d61c54afb8b30239c19276ce96 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Tue, 30 Dec 2025 09:57:44 +0100 Subject: [PATCH 33/36] Ensure required value error is shown for multiple upload widget --- src/onegov/form/widgets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/onegov/form/widgets.py b/src/onegov/form/widgets.py index 42dffb1b6b..772844849f 100644 --- a/src/onegov/form/widgets.py +++ b/src/onegov/form/widgets.py @@ -305,7 +305,10 @@ def __call__( """) if force_simple or len(field) == 0: - return simple_template.format(input_html) + return simple_template.format(input_html) + Markup('\n').join( + Markup('{}').format(error) + for error in field.errors + ) else: existing_html = Markup('').join( subfield( From f6f9d7c4ff6d3eaf238c0e10152c77f8eb6cdb25 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Tue, 30 Dec 2025 10:13:18 +0100 Subject: [PATCH 34/36] Extend tests with required upload field --- tests/onegov/form/test_fields.py | 1 + tests/onegov/form/test_parser.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 0f1f15b9fc..483dfb2dce 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -96,6 +96,7 @@ def create_field( assert data == {} assert field.file is None assert field.filename is None + assert field.validate(form) assert_whitelisted_mimetype_validator(field) form, field = create_field() diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index 429cf817d0..d62f8056ec 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -368,19 +368,19 @@ def test_parse_fileinput() -> None: assert form['file'].widget.multiple is False # type: ignore[attr-defined] # verify attached mime type validator - assert find_validator(form['file'], FileSizeLimit) assert form['file'].validators validator = find_validator(form['file'], WhitelistedMimeType) assert validator assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } + assert find_validator(form['file'], FileSizeLimit) - form = parse_form("File = *.*")() + form = parse_form("File *= *.*")() + assert form['file'].validators validator = find_validator(form['file'], WhitelistedMimeType) assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] - assert find_validator(form['file'], FileSizeLimit) # ensure nickname field did not get validators from the upload field @@ -397,17 +397,19 @@ def test_parse_multiplefileinput() -> None: assert form['files'].widget.multiple is True # type: ignore[attr-defined] # verify attached mime type validator + assert form['files'].validators validator = find_validator(form['files'], WhitelistedMimeType) assert validator assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } + assert find_validator(form['files'], FileSizeLimit) - form = parse_form("My files = *.* (multiple)")() + form = parse_form("My files *= *.* (multiple)")() + assert form['my_files'].validators validator = find_validator(form['my_files'], WhitelistedMimeType) assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] - assert find_validator(form['my_files'], FileSizeLimit) From c170834488d49f522cbc8379906cebbb66cec346 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Tue, 30 Dec 2025 10:31:06 +0100 Subject: [PATCH 35/36] Fix syntax error --- src/onegov/form/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 38f01a2b46..2dc3bcba11 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -169,7 +169,7 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: MIME_TYPES_EXCEL = { 'application/excel', - 'application/vnd.ms-excel,' + 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/vnd.ms-office', 'application/octet-stream', From 00b1614c4885bb7f6f90bf47611645a6b5dc2dd1 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 5 Jan 2026 09:32:30 +0100 Subject: [PATCH 36/36] Extend tests --- tests/onegov/form/test_parser.py | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index d62f8056ec..5d48f6c912 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -601,6 +601,59 @@ def test_dependent_validation() -> None: ) ) + # dependency with upload field + text = dedent(""" + method *= + (x) Drop off + Estimated drop off time = HH:MM + ( ) Upload + Attachment *= *.pdf|*.doc + """) + + form_class = parse_form(text) + + # drop off: no file required + form = form_class(MultiDict([('method', 'Drop off')])) + assert form.validate() + assert not form.errors + + # upload selected but no file provided + form = form_class(MultiDict([('method', 'Upload')])) + assert not form.validate() + assert form.errors == {'method_attachment': ['This field is required.']} + + # upload selected with large attachment and wrong type + from io import BytesIO + from werkzeug.datastructures import FileStorage + + fs_text = FileStorage( + stream=BytesIO(b'Hello, this is plain text\n' + + b'A' * 101 * 1000 ** 2), + filename='test.txt', + content_type='text/plain' + ) + form = form_class( + MultiDict([('method', 'Upload'), ('method_attachment', fs_text)])) + assert not form.validate() + assert form.errors == { + 'method_attachment': [ + 'Files of this type are not supported.', + 'The file is too large, please provide a file smaller ' + 'than 100.0 MB.' + ] + } + + # upload selected with large attachment and wrong type + fs_pdf = FileStorage( + stream=BytesIO(b'%PDF-1.4\n%'), + filename='test.pdf', + content_type='application/pdf' + ) + form = form_class( + MultiDict([('method', 'Upload'), ('method_attachment', fs_pdf)])) + assert form.validate() + assert not form.errors + def test_nested_regression() -> None: