Add unicode box characters

.github/workflows/macos.yml

@@ -26,11 +26,17 @@ jobs:
       run: |
         brew install llvm@14 tesseract remake
         python -m pip install --upgrade pip
-    - name: Install Mathics3 with full Python dependencies
+    - name: Install Mathics3 dependencies
       run: |
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install Mathics-Scanner[full]
-        python -m pip install -e "Mathics-Scanner[full] @ git+https://github.com/Mathics3/mathics-scanner"
+        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        cd mathics-scanner/
+        pip install -e .
+        bash -x admin-tools/make-JSON-tables.sh
+        cd ..
+    - name: Install Mathics3
+      run: |
         pip install -e .
         remake -x develop-full
     - name: Test Mathics3

.github/workflows/packages.yml

@@ -21,11 +21,15 @@ jobs:
     - name: Install OS dependencies
       run: |
         sudo apt-get update -qq && sudo apt-get install -qq liblapack-dev llvm-dev tesseract-ocr remake
-    - name: Install Mathics3 with full dependencies
+    - name: Install dependent Mathics3 programs
       run: |
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e "Mathics-Scanner[full] @ git+https://github.com/Mathics3/mathics-scanner"
+        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        cd mathics-scanner/
+        pip install -e .
+        bash -x admin-tools/make-JSON-tables.sh
+        cd ..
     - name: Run Mathics3 Combinatorica tests
       run: |
         git submodule init

.github/workflows/plot-tests.yml

@@ -21,19 +21,23 @@ jobs:
     - name: Install OS dependencies
       run: |
         sudo apt-get update -qq && sudo apt-get install -qq liblapack-dev llvm-dev tesseract-ocr remake
-    - name: Install Mathics3 with full dependencies
+    - name: Install dependent Mathics3 programs
       run: |
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
         python -m pip install -e "Mathics-Scanner[full] @ git+https://github.com/Mathics3/mathics-scanner"
         git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
         cd mathics-scanner/
         pip install -e .
+        bash -x ./admin-tools/make-JSON-tables.sh
         cd ..
         # We can comment out after next Mathics-Scanner release
         # python -m pip install Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+    - name: Install Mathics3 with full dependencies
+      run: |
         pip install -e .
+        bash -x ./admin-tools/make-JSON-tables.sh
         remake -x develop-full
     - name: Test Mathics
       run: |

.github/workflows/pyodide.yml

@@ -36,7 +36,9 @@ jobs:
           actions-cache-folder: emsdk-cache
 
       - name: Install pyodide-build
-        run: pip install pyodide-build
+        run: |
+          pip install -U setuptools
+          pip install pyodide-build
 
       - name: Set up Node.js
         uses: actions/setup-node@v4
@@ -54,9 +56,17 @@ jobs:
 
           pip install "setuptools>=70.0.0" PyYAML click packaging pytest
 
+      - name: Install dependent Mathics3 programs
+        run: |
           # We can comment out after next Mathics-Scanner release
-          python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner
-
+          git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+          cd mathics-scanner/
+          pip install -e .
+          bash -x admin-tools/make-JSON-tables.sh
+          cd ..
+      - name: Install Mathics3
+        run: |
           pip install --no-build-isolation -e .
-          make mathics/data/character-tables.json mathics/data/operator-tables.json
+          bash -x admin-tools/make-JSON-tables.sh
+          python -m pip install pytest
           make -j3 check

.github/workflows/ubuntu-bench.yml

@@ -21,12 +21,18 @@ jobs:
     - name: Install OS dependencies
       run: |
         sudo apt-get update -qq && sudo apt-get install -qq liblapack-dev llvm-dev tesseract-ocr remake
-    - name: Install Mathics3 with full dependencies
+    - name: Install dependent Mathics3 programs
       run: |
         python -m pip install --upgrade pip
         python -m pip install pytest-benchmark
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e "Mathics-Scanner[full] @ git+https://github.com/Mathics3/mathics-scanner"
+        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        cd mathics-scanner/
+        pip install -e .
+        bash -x admin-tools/make-JSON-tables.sh
+        cd ..
+    - name: Install Mathics3 with full dependencies
+      run: |
         # python -m pip install Mathics-Scanner[full]
         pip install -e .
         remake -x develop

.github/workflows/ubuntu-cython.yml

@@ -20,13 +20,15 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install Mathics3 dependencies
       run: |
         sudo apt-get update -qq && sudo apt-get install -qq liblapack-dev llvm-dev tesseract-ocr
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e "Mathics-Scanner[full] @ git+https://github.com/Mathics3/mathics-scanner"
+        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        cd mathics-scanner/
         pip install -e .
+        bash -x admin-tools/make-JSON-tables.sh
         cd ..
 
     - name: Install Mathics with full dependencies

.github/workflows/ubuntu.yml

@@ -21,19 +21,18 @@ jobs:
     - name: Install OS dependencies
       run: |
         sudo apt-get update -qq && sudo apt-get install -qq liblapack-dev llvm-dev tesseract-ocr remake
-    - name: Install Mathics3 with full dependencies
+    - name: Install Mathics3 dependencies
       run: |
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-        # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git
         cd mathics-scanner/
         pip install -e .
+        bash -x admin-tools/make-JSON-tables.sh
         cd ..
-        # We can comment out after next Mathics-Scanner release
-        # python -m pip install Mathics-Scanner[full]
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+    - name: Install Mathics3
+      run: |
         pip install -e .
         remake -x develop-full
     - name: Test Mathics

.github/workflows/windows.yml

@@ -33,11 +33,17 @@ jobs:
         set LLVM_DIR="C:\Program Files\LLVM"
     - name: Install Mathics3 with Python dependencies
       run: |
+        pip install pyocr # from full
         # We can comment out after next Mathics-Scanner release
         python -m pip install -e "Mathics-Scanner[full] @ git+https://github.com/Mathics3/mathics-scanner"
         pip install -e .
-
-        # python -m pip install Mathics-Scanner[full]
+        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        cd mathics-scanner/
+        pip install -e .
+        bash -x admin-tools/make-JSON-tables.sh
+        cd ..
+    - name: Install Mathics with full dependencies
+      run: |
         make develop-full
     - name: Test Mathics3
       # Limit pip install to a basic install *without* full dependencies.
@@ -48,7 +54,6 @@ jobs:
       #     we needs some CI that tests running when packages aren't available
       # So "dev" only below, not "dev,full".
       run: |
-        pip install pyocr # from full
         pip install -e .[dev]
         make pytest gstest
         make doctest DOCTEST_OPTIONS="--exclude WordCloud"

Makefile

@@ -69,17 +69,17 @@ build:
 # because pip install doesn't handle
 # INSTALL_REQUIRES properly
 #: Set up to run from the source tree
-develop:  mathics/data/character-tables.json mathics/data/operator-tables.json
+develop: mathics_scanner/data/boxing-characters.json mathics_scanner/data/named-characters.json mathics_scanner/data/operators.json
 	$(PIP) install -e .[dev]
 
 # See note above on ./setup.py
 #: Set up to run from the source tree with full dependencies
-develop-full:  mathics/data/character-tables.json mathics/data/operators.json
+develop-full: mathics_scanner/data/boxing-characters.json mathics_scanner/data/named-characters.json mathics_scanner/data/operators.json
 	$(PIP) install -e .[dev,full]
 
 # See note above on ./setup.py
 #: Set up to run from the source tree with full dependencies and Cython
-develop-full-cython: mathics/data/character-tables.json mathics/data/operators.json
+develop-full-cython: mathics_scanner/data/boxing-characters.json mathics_scanner/data/named-characters.json mathics_scanner/data/operators.json
 	$(PIP) install -e .[dev,full,cython]
 
 
@@ -126,7 +126,7 @@ clean: clean-cython clean-cache
 	   ($(MAKE) -C "$$dir" clean); \
 	done; \
 	rm -f factorials || true; \
-	rm -f mathics/data/character-tables.json || true; \
+	rm -f mathics/data/*.json || true; \
 	rm -rf build || true
 
 mypy:
@@ -166,7 +166,7 @@ latexdoc texdoc doc:
 	(cd mathics/doc/latex && $(MAKE) doc)
 
 #: Build JSON ASCII to unicode opcode table and operator table
-mathics/data/operator-tables.json mathics/data/character-tables.json mathics/data/operators.json:
+mathics_scanner/data/boxing-characters.json mathics_scanner/data/named-characters.json mathics_scanner/data/operators.json:
 	$(BASH) ./admin-tools/make-JSON-tables.sh
 
 #: Remove ChangeLog

admin-tools/make-JSON-tables.sh

@@ -5,7 +5,6 @@ mydir=$(dirname $bs)
 PYTHON=${PYTHON:-python}
 
 cd $mydir/../mathics/data
-mathics3-generate-json-table -o character-tables.json
-mathics3-generate-operator-json-table -o operator-tables.json
-# tokenizer looks for the table in the default place...
-mathics3-generate-operator-json-table
+mathics3-make-boxing-character-json -o boxing-characters.json
+mathics3-make-named-character-json -o named-characters.json
+mathics3-make-operator-json -o operators.json

mathics/builtin/forms/data.py

@@ -679,7 +679,7 @@ class StringForm(FormBaseClass):
      = `` is Global`a
 
     To use a 'Backquote' as a character, escape it with a backslash:
-    >> StringForm["`` is Global\`a", a]
+    >> StringForm["`` is Global\\`a", a]
      = a is Global`a
 
     Elements are formatted according the enclosing context:
@@ -699,7 +699,7 @@ class StringForm(FormBaseClass):
     }
     summary_text = "format a string from a template and a list of parameters"
 
-    def eval_makeboxes(self, s, args, form, evaluation):
+    def eval_makeboxes(self, s: String, args, form, evaluation: Evaluation):
         """MakeBoxes[StringForm[s_String, args___],
         form:StandardForm|TraditionalForm]"""
         try:

mathics/builtin/string/characters.py

@@ -7,9 +7,9 @@
 from mathics.core.atoms import String
 from mathics.core.attributes import A_LISTABLE, A_PROTECTED, A_READ_PROTECTED
 from mathics.core.builtin import Builtin, Test
-from mathics.core.convert.expression import to_mathics_list
 from mathics.core.evaluation import Evaluation
 from mathics.core.list import ListExpression
+from mathics.eval.string.characters import eval_Characters
 
 
 class Characters(Builtin):
@@ -28,12 +28,14 @@ class Characters(Builtin):
     """
 
     attributes = A_LISTABLE | A_PROTECTED
+    eval_error = Builtin.generic_argument_error
+    expected_args = 1
     summary_text = "list the characters in a string"
 
-    def eval(self, string, evaluation: Evaluation):
+    def eval(self, string: String, evaluation: Evaluation) -> ListExpression:
         "Characters[string_String]"
 
-        return to_mathics_list(*string.value, elements_conversion_fn=String)
+        return eval_Characters(string.value)
 
 
 class CharacterRange(Builtin):

mathics/builtin/string/charcodes.py

@@ -65,6 +65,8 @@ class ToCharacterCode(Builtin):
      = -Graphics-
     """
 
+    eval_error = Builtin.generic_argument_error
+    expected_args = (1, 2)
     summary_text = "convert a string to a list of character codes"
 
     def _encode(self, string, encoding, evaluation: Evaluation):

mathics/builtin/string/operations.py

@@ -360,6 +360,8 @@ class StringLength(Builtin):
     """
 
     attributes = A_LISTABLE | A_PROTECTED
+    eval_error = Builtin.generic_argument_error
+    expected_args = 1
 
     summary_text = "length of a string (in Unicode characters)"
 
@@ -863,6 +865,8 @@ class StringTake(Builtin):
         "take": 'Cannot take positions `1` through `2` in "`3`".',
     }
 
+    eval_error = Builtin.generic_argument_error
+    expected_args = 2
     summary_text = "sub-string from a range of positions"
 
     def eval(self, string: String, seqspec, evaluation: Evaluation):
@@ -894,7 +898,7 @@ def eval(self, string: String, seqspec, evaluation: Evaluation):
 
         return String(result[py_slice])
 
-    def eval_strings(self, strings, spec, evaluation):
+    def eval_strings(self, strings, spec, evaluation: Evaluation):
         "StringTake[strings__, spec_]"
         result_list = []
         for string in strings.elements:

mathics/core/convert/op.py

@@ -15,7 +15,7 @@
 
 
 # Load the conversion tables from disk
-characters_path = osp.join(ROOT_DIR, "data", "character-tables.json")
+characters_path = osp.join(ROOT_DIR, "data", "named-characters.json")
 assert osp.exists(
     characters_path
 ), f"ASCII operator to Unicode tables are missing from {characters_path}"

mathics/core/parser/operators.py

@@ -21,7 +21,7 @@
 
 # Load Mathics3 operator information from JSON. This file is derived from a
 # Mathics3 Operator Data YAML file in MathicsScanner.
-operator_tables_path = osp.join(ROOT_DIR, "data", "operator-tables.json")
+operator_tables_path = osp.join(ROOT_DIR, "data", "operators.json")
 assert osp.exists(
     operator_tables_path
 ), f"Internal error: Mathics3 Operator information are missing; expected to be in {operator_tables_path}"