From 5f4cf4e7d1d3d329a03b5fd3056f722248659bf9 Mon Sep 17 00:00:00 2001 From: James Prior Date: Sat, 8 Feb 2025 16:35:43 +0000 Subject: [PATCH 1/4] Pass env to Lexer --- liquid2/environment.py | 2 +- liquid2/lexer.py | 10 +++++++--- performance/benchmark_001.py | 7 ++++--- performance/profile_001.py | 3 ++- performance/profile_002.py | 3 ++- tests/test_lexer.py | 6 ++++-- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/liquid2/environment.py b/liquid2/environment.py index e3dafc2..80393ef 100644 --- a/liquid2/environment.py +++ b/liquid2/environment.py @@ -122,7 +122,7 @@ def setup_tags_and_filters(self) -> None: def tokenize(self, source: str) -> list[TokenT]: """Scan Liquid template _source_ and return a list of Markup objects.""" - lexer = self.lexer_class(source) + lexer = self.lexer_class(self, source) lexer.run() return lexer.markup diff --git a/liquid2/lexer.py b/liquid2/lexer.py index 59eadcb..8c02c33 100644 --- a/liquid2/lexer.py +++ b/liquid2/lexer.py @@ -30,6 +30,7 @@ from .token import is_token_type if TYPE_CHECKING: + from .environment import Environment from .token import TokenT @@ -178,6 +179,7 @@ class Lexer: TOKEN_RULES = _compile(NUMBERS, SYMBOLS, WORD) __slots__ = ( + "env", "in_range", "line_start", "line_statements", @@ -194,7 +196,9 @@ class Lexer: "template_string_stack", ) - def __init__(self, source: str) -> None: + def __init__(self, env: Environment, source: str) -> None: + self.env = env + self.markup: list[TokenT] = [] """Markup resulting from scanning a Liquid template.""" @@ -1168,8 +1172,8 @@ def lex_inside_liquid_block_comment(self) -> StateFn | None: return self.lex_inside_liquid_tag -def tokenize(source: str) -> list[TokenT]: +def tokenize(env: Environment, source: str) -> list[TokenT]: """Scan Liquid template _source_ and return a list of Markup objects.""" - lexer = Lexer(source) + lexer = Lexer(env, source) lexer.run() return lexer.markup diff --git a/performance/benchmark_001.py b/performance/benchmark_001.py index 3562024..1dc8589 100644 --- a/performance/benchmark_001.py +++ b/performance/benchmark_001.py @@ -17,9 +17,9 @@ def fixture(path_to_templates: Path) -> dict[str, str]: return loader_dict -def lex(templates: dict[str, str]) -> None: +def lex(env: Environment, templates: dict[str, str]) -> None: for source in templates.values(): - tokenize(source) + tokenize(env, source) def parse(env: Environment, templates: dict[str, str]) -> None: @@ -69,9 +69,10 @@ def benchmark(search_path: str, number: int = 1000, repeat: int = 5) -> None: print_result( "lex template (not expressions)", timeit.repeat( - "lex(templates)", + "lex(env, templates)", globals={ "lex": lex, + "env": Environment(), "search_path": search_path, "templates": templates, "tokenize": tokenize, diff --git a/performance/profile_001.py b/performance/profile_001.py index aac40e5..24a1514 100644 --- a/performance/profile_001.py +++ b/performance/profile_001.py @@ -45,10 +45,11 @@ def benchmark(search_path: str, number: int = 1000, repeat: int = 5) -> None: print_result( "scan template", timeit.repeat( - "tokenize(template)", + "tokenize(env, template)", globals={ "template": source, "tokenize": tokenize, + "env": Environment(), }, number=number, repeat=repeat, diff --git a/performance/profile_002.py b/performance/profile_002.py index 53efbdf..8e47c66 100644 --- a/performance/profile_002.py +++ b/performance/profile_002.py @@ -51,10 +51,11 @@ def benchmark(search_path: str, number: int = 1000, repeat: int = 5) -> None: print_result( "scan template", timeit.repeat( - "tokenize(template)", + "tokenize(env, template)", globals={ "template": source, "tokenize": tokenize, + "env": Environment(), }, number=number, repeat=repeat, diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 1778216..ac2221f 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -3,7 +3,7 @@ import pytest -from liquid2 import tokenize +from liquid2 import DEFAULT_ENVIRONMENT @dataclass @@ -222,4 +222,6 @@ class Case: @pytest.mark.parametrize("case", TEST_CASES, ids=operator.attrgetter("name")) def test_lexer(case: Case) -> None: - assert "".join(str(t) for t in tokenize(case.source)) == case.want + assert ( + "".join(str(t) for t in DEFAULT_ENVIRONMENT.tokenize(case.source)) == case.want + ) From 384849af44da97c23fa03ffc32b49daf61b09885 Mon Sep 17 00:00:00 2001 From: James Prior Date: Sat, 8 Feb 2025 16:55:04 +0000 Subject: [PATCH 2/4] Add `shorthand_indexes` option --- CHANGELOG.md | 10 ++++ liquid2/__about__.py | 2 +- liquid2/environment.py | 4 ++ liquid2/lexer.py | 9 +++- tests/test_shorthand_indexes.py | 81 +++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 tests/test_shorthand_indexes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ce10d..c7d89ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Python Liquid2 Change Log +## Version 0.4.0 + +**Features** + +- Added the `shorthand_indexes` class variable to `liquid2.Environment`. When `shorthand_indexes` is set to `True` (the default), array indexes in variable paths need not be surrounded by square brackets. + +**Changes** + +- `liquid2.tokenize` and `liquid2.lexer.Lexer` now require the current `Environment` to be passed as the first argument. + ## Version 0.3.0 **Breaking changes** diff --git a/liquid2/__about__.py b/liquid2/__about__.py index 493f741..6a9beea 100644 --- a/liquid2/__about__.py +++ b/liquid2/__about__.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/liquid2/environment.py b/liquid2/environment.py index 80393ef..a9031b5 100644 --- a/liquid2/environment.py +++ b/liquid2/environment.py @@ -74,6 +74,10 @@ class Environment: """If True (the default), indicates that blocks rendering to whitespace only will not be output.""" + shorthand_indexes: bool = True + """If True (the default), array indexes can be separated by dots without enclosing + square brackets.""" + lexer_class = Lexer """The lexer class to use when scanning template source text.""" diff --git a/liquid2/lexer.py b/liquid2/lexer.py index 8c02c33..a0d33b5 100644 --- a/liquid2/lexer.py +++ b/liquid2/lexer.py @@ -318,8 +318,15 @@ def accept_path(self, *, carry: bool = False) -> None: self.pos += match.end() - match.start() self.start = self.pos self.path_stack[-1].stop = self.pos + elif self.env.shorthand_indexes: + if match := self.RE_INDEX.match(self.source, self.pos): + self.path_stack[-1].path.append(int(match.group())) + self.pos += match.end() - match.start() + self.start = self.pos + else: + self.error("array indexes must use bracket notation") else: - self.error("expected a property name") + self.error("expected a property name or array index") elif c == "]": if len(self.path_stack) == 1: diff --git a/tests/test_shorthand_indexes.py b/tests/test_shorthand_indexes.py new file mode 100644 index 0000000..db259d8 --- /dev/null +++ b/tests/test_shorthand_indexes.py @@ -0,0 +1,81 @@ +import pytest + +from liquid2 import Environment +from liquid2.exceptions import LiquidSyntaxError + + +class MockEnv(Environment): + shorthand_indexes = False + + +def test_disable_shorthand_indexes() -> None: + env = MockEnv() + with pytest.raises(LiquidSyntaxError): + env.from_string("{{ foo.0.bar }}") + + +ENV = Environment() + + +def test_shorthand_index() -> None: + data = {"foo": ["World", "Liquid"]} + template = ENV.from_string("Hello, {{ foo.0 }}!") + assert template.render(**data) == "Hello, World!" + template = ENV.from_string("Hello, {{ foo.1 }}!") + assert template.render(**data) == "Hello, Liquid!" + + +def test_consecutive_shorthand_indexes() -> None: + data = {"foo": [["World", "Liquid"]]} + template = ENV.from_string("Hello, {{ foo.0.0 }}!") + assert template.render(**data) == "Hello, World!" + template = ENV.from_string("Hello, {{ foo.0.1 }}!") + assert template.render(**data) == "Hello, Liquid!" + + +def test_shorthand_index_dot_property() -> None: + data = {"foo": [{"bar": "World"}, {"bar": "Liquid"}]} + template = ENV.from_string("Hello, {{ foo.0.bar }}!") + assert template.render(**data) == "Hello, World!" + template = ENV.from_string("Hello, {{ foo.1.bar }}!") + assert template.render(**data) == "Hello, Liquid!" + + +def test_shorthand_index_in_loop_expression() -> None: + data = {"foo": [["World", "Liquid"]]} + template = ENV.from_string("{% for x in foo.0 %}Hello, {{ x }}! {% endfor %}") + assert template.render(**data) == "Hello, World! Hello, Liquid! " + + +def test_shorthand_index_in_conditional_expression() -> None: + data = {"foo": ["World", "Liquid"]} + template = ENV.from_string("{% if foo.0 %}Hello, {{ foo.0 }}!{% endif %}") + assert template.render(**data) == "Hello, World!" + template = ENV.from_string("{% if foo.2 %}Hello, {{ foo.2 }}!{% endif %}") + assert template.render(**data) == "" + + +def test_shorthand_indexes_in_case_tag() -> None: + data = {"foo": ["World", "Liquid"]} + template = ENV.from_string( + "{% case foo.0 %}{% when 'World' %}Hello, World!{% endcase %}" + ) + assert template.render(**data) == "Hello, World!" + + +def test_shorthand_indexes_in_ternary_expressions() -> None: + data = {"foo": ["World", "Liquid"]} + template = ENV.from_string("Hello, {{ foo.0 }}!") + assert template.render(**data) == "Hello, World!" + template = ENV.from_string("Hello, {{ 'you' if foo.1 }}!") + assert template.render(**data) == "Hello, you!" + template = ENV.from_string("Hello, {{ 'you' if foo.99 else foo.1 }}!") + assert template.render(**data) == "Hello, Liquid!" + + +def test_shorthand_indexes_in_logical_not_expressions() -> None: + data = {"foo": ["World", "Liquid"]} + template = ENV.from_string("{% if not foo.0 %}Hello, {{ foo.0 }}!{% endif %}") + assert template.render(**data) == "" + template = ENV.from_string("{% if not foo.2 %}Hello, {{ foo.0 }}!{% endif %}") + assert template.render(**data) == "Hello, World!" From 75d3d706dd91b6f7d238d1c942d10d801287f9bb Mon Sep 17 00:00:00 2001 From: James Prior Date: Sat, 8 Feb 2025 17:01:01 +0000 Subject: [PATCH 3/4] Mention `shorthand_indexes` in the migration guide --- docs/migration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/migration.md b/docs/migration.md index d9960a1..a813de0 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -168,6 +168,10 @@ Integer and float literals can use scientific notation, like `1.2e3` or `1e-2`. Filter and tag named arguments can be separated by a `:` or `=`. Previously only `:` was allowed. +### Shorthand array indexes + +By default, array indexes in paths to variables are allowed to use shorthand dotted notation. `{{ foo.0.bar }}` is equivalent to `{{ foo[0].bar }}`. This can be disabled with the `shorthand_indexes` class variable on an `Environment` subclass. + ### Template inheritance ([docs](tag_reference.md#extends)) From dfc520934e62cfc00de8bc727668f6cb26752c6f Mon Sep 17 00:00:00 2001 From: James Prior Date: Thu, 13 Feb 2025 17:10:36 +0000 Subject: [PATCH 4/4] Disable shorthand indexes by default --- CHANGELOG.md | 4 ++-- docs/migration.md | 2 +- liquid2/environment.py | 6 +++--- tests/test_shorthand_indexes.py | 14 +++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7d89ba..6e70ab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,10 @@ # Python Liquid2 Change Log -## Version 0.4.0 +## Version 0.4.0 (unreleased) **Features** -- Added the `shorthand_indexes` class variable to `liquid2.Environment`. When `shorthand_indexes` is set to `True` (the default), array indexes in variable paths need not be surrounded by square brackets. +- Added the `shorthand_indexes` class variable to `liquid2.Environment`. When `shorthand_indexes` is set to `True` (the default is `False`), array indexes in variable paths need not be surrounded by square brackets. **Changes** diff --git a/docs/migration.md b/docs/migration.md index a813de0..5176b7b 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -170,7 +170,7 @@ Filter and tag named arguments can be separated by a `:` or `=`. Previously only ### Shorthand array indexes -By default, array indexes in paths to variables are allowed to use shorthand dotted notation. `{{ foo.0.bar }}` is equivalent to `{{ foo[0].bar }}`. This can be disabled with the `shorthand_indexes` class variable on an `Environment` subclass. +Optionally allow shorthand dotted notation for array indexes in paths to variables. When the `Environment` class variable `shorthand_indexes` is set to `True` (default is `False`), `{{ foo.0.bar }}` is equivalent to `{{ foo[0].bar }}`. ### Template inheritance diff --git a/liquid2/environment.py b/liquid2/environment.py index a9031b5..a479f08 100644 --- a/liquid2/environment.py +++ b/liquid2/environment.py @@ -74,9 +74,9 @@ class Environment: """If True (the default), indicates that blocks rendering to whitespace only will not be output.""" - shorthand_indexes: bool = True - """If True (the default), array indexes can be separated by dots without enclosing - square brackets.""" + shorthand_indexes: bool = False + """If True, array indexes can be separated by dots without enclosing square + brackets. The default is `False`.""" lexer_class = Lexer """The lexer class to use when scanning template source text.""" diff --git a/tests/test_shorthand_indexes.py b/tests/test_shorthand_indexes.py index db259d8..dd39620 100644 --- a/tests/test_shorthand_indexes.py +++ b/tests/test_shorthand_indexes.py @@ -4,17 +4,17 @@ from liquid2.exceptions import LiquidSyntaxError -class MockEnv(Environment): - shorthand_indexes = False - - -def test_disable_shorthand_indexes() -> None: - env = MockEnv() +def test_shorthand_indexes_are_disabled_by_default() -> None: + env = Environment() with pytest.raises(LiquidSyntaxError): env.from_string("{{ foo.0.bar }}") -ENV = Environment() +class MockEnv(Environment): + shorthand_indexes = True + + +ENV = MockEnv() def test_shorthand_index() -> None: