From 15dd2d64c81d3e29950dc8cf10420d49d8137d1d Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 13:00:00 -0400 Subject: [PATCH 1/8] RawComma -> Comma Also, Pattern operator is right associative. --- mathics_scanner/data/operators.yml | 2 +- mathics_scanner/tokeniser.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mathics_scanner/data/operators.yml b/mathics_scanner/data/operators.yml index fe667ab6..07d9e723 100644 --- a/mathics_scanner/data/operators.yml +++ b/mathics_scanner/data/operators.yml @@ -4731,7 +4731,7 @@ Pattern: FullForm: Pattern[symb, expr] arity: Binary affix: Infix - associativity: "unknown" + associativity: right meaningful: true # comments: diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index 2441f55d..97359f8a 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -178,7 +178,6 @@ "Postfix": "SlashSlash", "Prefix": "At", "Put": "GreaterGreater", - "RawComma": "Comma", "RawLeftBrace": "OpenCurly", "RawLeftBracket": "OpenSquare", "RawRightBrace": "CloseCurly", @@ -233,6 +232,7 @@ def init_module(): tokens: List[Tuple[str, ...]] = [ ("BoxInputEscape", r" \\[*]"), + ("Comma", r" \, "), ("Definition", r"\? "), ("Get", r"\<\<"), ("QuestionQuestion", r"\?\? "), @@ -242,7 +242,6 @@ def init_module(): ("Pattern", pattern_pattern), ("Put", r"\>\>"), ("PutAppend", r"\>\>\>"), - ("RawComma", r" \, "), ("LessBar", r" \<\| "), ("OpenCurly", r" \{ "), ("RawLeftBracket", r" \[ "), @@ -444,7 +443,7 @@ def init_module(): ")": ("CloseParen",), "*": ("NonCommutativeMultiply", "TimesBy", "Times"), "+": ("Increment", "AddTo", "Plus"), - ",": ("RawComma",), + ",": ("Comma",), "-": ( # Note that "Minus" has to come last. "Decrement", From 1d3ac9fa0cbd8bbd17dc1eaab7bf384a4c315958 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 13:13:22 -0400 Subject: [PATCH 2/8] Tiny simplification --- mathics_scanner/tokeniser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index 97359f8a..5ac0513d 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -178,7 +178,6 @@ "Postfix": "SlashSlash", "Prefix": "At", "Put": "GreaterGreater", - "RawLeftBrace": "OpenCurly", "RawLeftBracket": "OpenSquare", "RawRightBrace": "CloseCurly", "RawRightBracket": "CloseSquare", From 4ad8fa78203c048ab58cc56bb4e0c851a86a3f31 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 15:22:58 -0400 Subject: [PATCH 3/8] RawLeftBracket -> OpenSquare --- mathics_scanner/tokeniser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index 5ac0513d..d7d8b73a 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -178,7 +178,6 @@ "Postfix": "SlashSlash", "Prefix": "At", "Put": "GreaterGreater", - "RawLeftBracket": "OpenSquare", "RawRightBrace": "CloseCurly", "RawRightBracket": "CloseSquare", "RightRowBox": "CloseParen", @@ -243,8 +242,8 @@ def init_module(): ("PutAppend", r"\>\>\>"), ("LessBar", r" \<\| "), ("OpenCurly", r" \{ "), - ("RawLeftBracket", r" \[ "), ("OpenParen", r" \( "), + ("OpenSquare", r" \[ "), ("BarGreater", r" \|\> "), ("CloseCurly", r" \} "), ("RawRightBracket", r" \] "), @@ -506,7 +505,7 @@ def init_module(): "PatternTest", ), "@": ("ApplyList", "Apply", "Composition", "Prefix"), - "[": ("RawLeftBracket",), + "[": ("OpenSquare",), "\\": ( # Note that "RawBackSlash" has to come last. "BoxInputEscape", From cbeb9bda54d98fbb306b12e59a4d00e3ab679734 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 15:26:39 -0400 Subject: [PATCH 4/8] "RawRightBracket" -> "CloseSquare" --- mathics_scanner/tokeniser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index d7d8b73a..c3726725 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -179,7 +179,6 @@ "Prefix": "At", "Put": "GreaterGreater", "RawRightBrace": "CloseCurly", - "RawRightBracket": "CloseSquare", "RightRowBox": "CloseParen", "Rule": "MinusGreater", "RuleDelayed": "ColonGreater", @@ -246,7 +245,7 @@ def init_module(): ("OpenSquare", r" \[ "), ("BarGreater", r" \|\> "), ("CloseCurly", r" \} "), - ("RawRightBracket", r" \] "), + ("CloseSquare", r" \] "), ("CloseParen", r" \) "), ("Slot", slot_pattern), ("SlotSequence", r"\#\#\d*"), @@ -523,7 +522,7 @@ def init_module(): "FormBox", "RawBackslash", ), - "]": ("RawRightBracket",), + "]": ("CloseSquare",), "^": ( # Note that "Power" has to come last. "UpSetDelayed", From ba6cb4135ffa17cb9a1b769076718979a908f003 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 15:33:30 -0400 Subject: [PATCH 5/8] {Left,Right}RawBracket -> {Open,Close}Square --- mathics_scanner/tokeniser.py | 2 -- test/test_tokeniser.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index c3726725..fef27949 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -178,8 +178,6 @@ "Postfix": "SlashSlash", "Prefix": "At", "Put": "GreaterGreater", - "RawRightBrace": "CloseCurly", - "RightRowBox": "CloseParen", "Rule": "MinusGreater", "RuleDelayed": "ColonGreater", "ReplaceAll": "SlashDot", diff --git a/test/test_tokeniser.py b/test/test_tokeniser.py index bd28a2a2..357fe26c 100644 --- a/test/test_tokeniser.py +++ b/test/test_tokeniser.py @@ -137,9 +137,9 @@ def test_boxes(): Token("LeftRowBox", "\\(", 0), Token("BoxInputEscape", "\\*", 2), Token("Symbol", "RowBox", 4), - Token("RawLeftBracket", "[", 10), + Token("OpenSquare", "[", 10), Token("Symbol", "a", 11), - Token("RawRightBracket", "]", 12), + Token("CloseSquare", "]", 12), Token("RightRowBox", "\\)", 13), ] From aea846792bbd8eb274aa93eb9416f3977156cc1c Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 16:14:34 -0400 Subject: [PATCH 6/8] Version change were 10-ish now. Match version of Mathics-core more closely --- mathics_scanner/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathics_scanner/version.py b/mathics_scanner/version.py index 446464f0..6e486294 100644 --- a/mathics_scanner/version.py +++ b/mathics_scanner/version.py @@ -4,4 +4,4 @@ # well as importing into Python. That's why there is no # space around "=" below. # fmt: off -__version__="2.0.1.dev0" # noqa +__version__="10.0.0.dev0" # noqa From 7fbf4a3c4c9a0ff9fc308d593a3562e315cd6ceb Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 16:24:50 -0400 Subject: [PATCH 7/8] Update to v6 to squelch CI warnings --- .github/workflows/isort-and-black-checks.yml | 4 ++-- .github/workflows/macos.yml | 4 ++-- .github/workflows/mathics3-doctest.yml | 6 +++--- .github/workflows/pyodide.yml | 4 ++-- .github/workflows/ubuntu.yml | 4 ++-- .github/workflows/windows.yml | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/isort-and-black-checks.yml b/.github/workflows/isort-and-black-checks.yml index 968c21a9..1ceb4f54 100644 --- a/.github/workflows/isort-and-black-checks.yml +++ b/.github/workflows/isort-and-black-checks.yml @@ -9,9 +9,9 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python 3.13 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.13 - name: Install click, black, and isort diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 626e7e2f..f8a792c6 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -14,9 +14,9 @@ jobs: os: [macOS] python-version: ['3.10', '3.12', '3.13'] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/mathics3-doctest.yml b/.github/workflows/mathics3-doctest.yml index fe8fe047..080929bf 100644 --- a/.github/workflows/mathics3-doctest.yml +++ b/.github/workflows/mathics3-doctest.yml @@ -13,9 +13,9 @@ jobs: matrix: python-version: ['3.13'] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install OS dependencies @@ -36,7 +36,7 @@ jobs: - name: Build Mathics3 run: | # Until next Mathics3/mathics-core release is out... - git clone --depth 1 https://github.com/Mathics3/mathics-core.git + git clone --depth 1 -b scanner-name-changes https://github.com/Mathics3/mathics-core.git cd mathics-core/ python -m pip install -e .[dev] cp -v ../mathics_scanner/data/boxing-characters.json mathics/data/boxing-characters.json diff --git a/.github/workflows/pyodide.yml b/.github/workflows/pyodide.yml index 17e6d20a..5bd83df7 100644 --- a/.github/workflows/pyodide.yml +++ b/.github/workflows/pyodide.yml @@ -22,10 +22,10 @@ jobs: NODE_VERSION: 20 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 4a20d500..f35b78af 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -13,9 +13,9 @@ jobs: matrix: python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install Mathics3 scanner without JSON files diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index ca8de1e6..09dfad56 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -14,9 +14,9 @@ jobs: os: [windows] python-version: ['3.10', '3.13'] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From b76cb78109c2be54edbce12eabc36e920acee4e0 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 15 Mar 2026 19:26:45 -0400 Subject: [PATCH 8/8] Reinstate token name changes --- .github/workflows/mathics3-doctest.yml | 2 +- Makefile | 2 +- mathics_scanner/data/operators.yml | 7 ++++++- mathics_scanner/tokeniser.py | 5 +++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mathics3-doctest.yml b/.github/workflows/mathics3-doctest.yml index 080929bf..42ddfc6c 100644 --- a/.github/workflows/mathics3-doctest.yml +++ b/.github/workflows/mathics3-doctest.yml @@ -45,4 +45,4 @@ jobs: cd .. - name: Run Mathics3 tests run: | - remake -x check-mathics + remake -x check-mathics3 diff --git a/Makefile b/Makefile index e6de2cd9..0bd56995 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ RM ?= rm PIP_INSTALL_OPTS ?= .PHONY: all build \ - check check-full check-mathics clean \ + check check-full check-mathics3 clean \ develop dist doc \ inputrc-no-unicode \ inputrc-unicode \ diff --git a/mathics_scanner/data/operators.yml b/mathics_scanner/data/operators.yml index 07d9e723..b0dd7cf4 100644 --- a/mathics_scanner/data/operators.yml +++ b/mathics_scanner/data/operators.yml @@ -4731,7 +4731,12 @@ Pattern: FullForm: Pattern[symb, expr] arity: Binary affix: Infix - associativity: right + # Should be right associative, + # but something in implicit multiplication + # is messing things up? In: + # ConditionalExpression[expr1_, cond_] expr2_ + # ? + associativity: "unknown" meaningful: true # comments: diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index fef27949..9d8d7e13 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -178,6 +178,7 @@ "Postfix": "SlashSlash", "Prefix": "At", "Put": "GreaterGreater", + "RawComma": "Comma", "Rule": "MinusGreater", "RuleDelayed": "ColonGreater", "ReplaceAll": "SlashDot", @@ -227,7 +228,6 @@ def init_module(): tokens: List[Tuple[str, ...]] = [ ("BoxInputEscape", r" \\[*]"), - ("Comma", r" \, "), ("Definition", r"\? "), ("Get", r"\<\<"), ("QuestionQuestion", r"\?\? "), @@ -237,6 +237,7 @@ def init_module(): ("Pattern", pattern_pattern), ("Put", r"\>\>"), ("PutAppend", r"\>\>\>"), + ("RawComma", r" \, "), ("LessBar", r" \<\| "), ("OpenCurly", r" \{ "), ("OpenParen", r" \( "), @@ -438,7 +439,7 @@ def init_module(): ")": ("CloseParen",), "*": ("NonCommutativeMultiply", "TimesBy", "Times"), "+": ("Increment", "AddTo", "Plus"), - ",": ("Comma",), + ",": ("RawComma",), "-": ( # Note that "Minus" has to come last. "Decrement",