Use Unicode box characters inside strings

.github/workflows/mathics3-doctest.yml

@@ -0,0 +1,48 @@
+name: Mathics3 Doctest (Mathics3 doctest)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.13']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install OS dependencies
+      run: |
+        sudo apt-get update -qq && sudo apt-get install -qq liblapack-dev llvm-dev tesseract-ocr remake
+    - name: Install Mathics3 scanner without JSON
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+    - name: Install JSON files
+      run: |
+        python -m mathics_scanner.generate.boxing_characters -o mathics_scanner/data/boxing-characters.json
+        ls -l mathics_scanner/data/boxing-characters.json
+        python -m mathics_scanner.generate.named_characters -o mathics_scanner/data/named-characters.json
+        ls -l mathics_scanner/data/named-characters.json
+        python -m mathics_scanner.generate.operators -o mathics_scanner/data/operators.json
+        ls -l mathics_scanner/data/operators.json
+    - name: Build Mathics3
+      run: |
+        # Until next Mathics3/mathics-core release is out...
+        git clone -b add-unicode-box-characters --depth 1 https://github.com/Mathics3/mathics-core.git
+        cd mathics-core/
+        python -m pip install -e .[dev]
+        cp -v ../mathics_scanner/data/boxing-characters.json mathics/data/boxing-characters.json
+        cp -v ../mathics_scanner/data/named-characters.json mathics/data/named-characters.json
+        cp -v ../mathics_scanner/data/operators.json mathics/data/operators.json
+        cd ..
+    - name: Run Mathics3 tests
+      run: |
+        remake -x check-mathics

.github/workflows/osx.yml

@@ -1,4 +1,4 @@
-name: Mathics_Script (OSX)
+name: Mathics3 Scanner (OSX)
 
 on:
   push:
@@ -23,12 +23,17 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e .
-    - name: Install Mathics Scanner
+    - name: Install Mathics3 scanner without JSON files
       run: |
-        make
+        python -m pip install --upgrade pip
+        pip install -e .
+    - name: Install JSON files
+      run: |
+        python -m mathics_scanner.generate.boxing_characters
+        python -m mathics_scanner.generate.named_characters
+        python -m mathics_scanner.generate.operators
     - name: Test Mathics3 Scanner
       run: |
         pip install -r requirements-dev.txt
         pip install -r requirements-full.txt
-        python -m mathics_scanner.generate.build_tables
         make check

.github/workflows/pyodide.yml

@@ -43,7 +43,7 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
 
-      - name: Set up Pyodide virtual environment and run tests
+      - name: Set up Pyodide virtual environment
         run: |
           # Set up Pyodide virtual environment
           pyodide xbuildenv install ${{ env.PYODIDE_VERSION }}

.github/workflows/ubuntu.yml

@@ -1,4 +1,4 @@
-name: Mathics_Script (ubuntu)
+name: Mathics3 Scanner (ubuntu)
 
 on:
   push:
@@ -18,16 +18,17 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install Mathics3 scanner without JSON files
       run: |
         python -m pip install --upgrade pip
         pip install -e .
-    - name: Install Mathics_Scanner
+    - name: Install JSON files
       run: |
-        make
+        python -m mathics_scanner.generate.boxing_characters
+        python -m mathics_scanner.generate.named_characters
+        python -m mathics_scanner.generate.operators
     - name: Test Mathics3 Scanner
       run: |
         pip install -r requirements-dev.txt
         pip install -r requirements-full.txt
-        python -m mathics_scanner.generate.build_tables
         make check

.github/workflows/windows.yml

@@ -1,4 +1,4 @@
-name: Mathics (Windows)
+name: Mathics3 Scanner (Windows)
 
 on:
   push:
@@ -23,13 +23,17 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e .
-    - name: Install Mathics Scanner
+    - name: Install Mathics3 scanner without JSON files
       run: |
+        python -m pip install --upgrade pip
         pip install -e .
-    - name: Test Mathics3
+    - name: Install JSON files
+      run: |
+        python -m mathics_scanner.generate.boxing_characters
+        python -m mathics_scanner.generate.named_characters
+        python -m mathics_scanner.generate.operators
+    - name: Test Mathics3 Scanner
       run: |
-        # Ideally we should not have to do this.
-        python mathics_scanner/generate/build_tables.py
-        python mathics_scanner/generate/build_operator_tables.py
-        pip install -e .[dev,full]
-        py.test test
+        pip install -r requirements-dev.txt
+        pip install -r requirements-full.txt
+        make check

Makefile

@@ -22,21 +22,22 @@ PIP_INSTALL_OPTS ?=
 #: Default target - same as "develop"
 all: develop
 
-mathics_scanner/data/character-tables.json: mathics_scanner/data/named-characters.yml
-	$(PIP) install -r requirements-dev.txt
-	$(PYTHON) mathics_scanner/generate/build_tables.py
+mathics_scanner/data/boxing-characters.json: mathics_scanner/data/boxing-characters.yml
+	$(PYTHON) mathics_scanner/generate/boxing_characters.py
+
+mathics_scanner/data/named-characters.json: mathics_scanner/data/named-characters.yml
+	$(PYTHON) mathics_scanner/generate/named_characters.py
 
 mathics_scanner/data/operators.json: mathics_scanner/data/operators.yml
-	$(PIP) install -r requirements-dev.txt
-	$(PYTHON) mathics_scanner/generate/build_operator_tables.py
+	$(PYTHON) mathics_scanner/generate/operators.py
 
 #: build everything needed to install
-build: mathics_scanner/data/characters.json mathics_scanner/data/operators.json
+build: mathics_scanner/data/characters.json mathics_scanner/data/named_characters.json mathics_scanner/data/operators.json
 	$(PYTHON) ./setup.py build
 
 #: Set up to run from the source tree
-develop: mathics_scanner/data/character-tables.json mathics_scanner/data/operators.json
-	$(PIP) install -e .$(PIP_INSTALL_OPTS)
+develop: mathics_scanner/data/boxing-characters.json mathics_scanner/data/named-characters.json mathics_scanner/data/operators.json
+	$(PIP) install --no-build-isolation -e . $(PIP_INSTALL_OPTS)
 
 #: Build distribution
 dist: admin-tools/make-dist.sh
@@ -56,16 +57,16 @@ check: pytest
 test: check
 
 #: Build Sphinx HTML documentation
-doc:  mathics_scanner/data/character-tables.json
+doc:  mathics_scanner/data/named-characters.json
 	make -C docs html
 
 #: Remove derived files
 clean:
 	@find . -name *.pyc -type f -delete; \
-	$(RM) -f mathics_scanner/data/character-tables.json mathics_scanner/data/operators.json || true
+	$(RM) -f mathics_scanner/data/*.json || true
 
 #: Run py.test tests. Use environment variable "o" for pytest options
-pytest: mathics_scanner/data/character-tables.json
+pytest: mathics_scanner/data/named-characters.json
 	$(PYTHON) -m pytest test $o
 
 #: Print to stdout a GNU Readline inputrc without Unicode
@@ -77,7 +78,7 @@ inputrc-unicode:
 	$(PYTHON) -m mathics_scanner.generate.rl_inputrc inputrc-unicode
 
 #: Run Mathics core checks
-check-mathics::
+check-mathics:
 	MATHICS_CHARACTER_ENCODING="ASCII" $(PYTHON) -m mathics.docpipeline $o
 	pytest test/test_mathics_precedence.py
 

admin-tools/make-JSON-tables.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
-# Create a complete set of tables.
-# This just runs build_tables.py in this distribution
+# Create a complete set of JSON tables.
 bs=${BASH_SOURCE[0]}
 mydir=$(dirname $bs)
 PYTHON=${PYTHON:-python}
 
 cd $mydir/../mathics_scanner/data
-$PYTHON ../generate/build_tables.py -o character-tables.json
-$PYTHON ../generate/build_operator_tables.py -o operators.json
+$PYTHON ../generate/boxing_characters.py -o boxing-characters.json
+$PYTHON ../generate/named_characters.py -o named-characters.json
+$PYTHON ../generate/operators.py -o operators.json

mathics_scanner/__init__.py

@@ -7,8 +7,8 @@
 """
 
 from mathics_scanner.characters import (
+    NAMED_CHARACTERS,
     aliased_characters,
-    named_characters,
     replace_unicode_with_wl,
     replace_wl_with_plain_text,
 )
@@ -34,14 +34,14 @@
     "InvalidSyntaxError",
     "LineFeeder",
     "MultiLineFeeder",
+    "NAMED_CHARACTERS",
     "SyntaxError",
     "SingleLineFeeder",
     # "Token",
     # "Tokeniser",
     "__version__",
     "aliased_characters",
     # "is_symbol_name",
-    "named_characters",
     "replace_unicode_with_wl",
     "replace_wl_with_plain_text",
 ]