Operator corrections, yet again!

.github/workflows/mathics3-doctest.yml

@@ -36,12 +36,9 @@ jobs:
     - name: Build Mathics3
       run: |
         # Until next Mathics3/mathics-core release is out...
-        git clone --depth 1 https://github.com/Mathics3/mathics-core.git
+        git clone --depth 1 -b more-scanner-changes https://github.com/Mathics3/mathics-core.git
         cd mathics-core/
         python -m pip install -e .[dev]
-        cp -v ../mathics_scanner/data/boxing-characters.json mathics/data/boxing-characters.json
-        cp -v ../mathics_scanner/data/named-characters.json mathics/data/named-characters.json
-        cp -v ../mathics_scanner/data/operators.json mathics/data/operators.json
         cd ..
     - name: Run Mathics3 tests
       run: |

mathics_scanner/data/named-characters.yml

@@ -5970,6 +5970,12 @@ Implies:
   wl-reference: https://reference.wolfram.com/language/ref/character/Implies.html
   wl-unicode: "\uF523"
 
+Increment:
+  ascii: "++"
+  has-unicode-inverse: false
+  is-letter-like: false
+  operator-name: Increment
+
 IndentingNewLine:
   esc-alias: nl
   has-unicode-inverse: false
@@ -5999,12 +6005,6 @@ Infix:
   has-unicode-inverse: false
   is-letter-like: false
 
-Information:
-  ascii: "??"
-  has-unicode-inverse: false
-  is-letter-like: false
-  operator-name: Information
-
 Integral:
   amslatex: '\int'
   esc-alias: int
@@ -7794,6 +7794,9 @@ OptionKey:
   wl-reference: https://reference.wolfram.com/language/ref/character/OptionKey.html
   wl-unicode: "\uF7D2"
 
+# Optional and Pattern are both operators.  When ":" represents one or
+# another depends on context, and determining which is what is
+# mysterious right now.
 Optional:
   ascii: ":"
   has-unicode-inverse: false
@@ -7868,6 +7871,15 @@ PartialD:
   wl-unicode: "\u2202"
   wl-unicode-name: PARTIAL DIFFERENTIAL
 
+# Optional and Pattern are both operators.  When ":" represents one or
+# another depends on context, and determining which is what is
+# mysterious right now.
+Pattern:
+  ascii: ":"
+  has-unicode-inverse: false
+  is-letter-like: false
+  operator-name: Pattern
+
 # See also RawQuestion
 PatternTest:
   ascii: "?"
@@ -8134,6 +8146,12 @@ QuarterNote:
   wl-unicode: "\u2669"
   wl-unicode-name: QUARTER NOTE
 
+QuestionQuestion:
+  ascii: "??"
+  has-unicode-inverse: false
+  is-letter-like: false
+  operator-name: QuestionQuestion
+
 RHacek:
   esc-alias: rv
   has-unicode-inverse: false
@@ -9738,6 +9756,18 @@ SkeletonIndicator:
   wl-unicode: "\u2043"
   wl-unicode-name: HYPHEN BULLET
 
+Slot:
+  ascii: "#"
+  has-unicode-inverse: false
+  is-letter-like: false
+  operator-name: Slot
+
+SlotSequence:
+  ascii: "##"
+  has-unicode-inverse: false
+  is-letter-like: false
+  operator-name: SlotSequence
+
 SmallCircle:
   amslatex: '\circ'
   esc-alias: sc

mathics_scanner/data/operators.yml

@@ -4457,6 +4457,9 @@ NumberPrecisionPostfix:
   meaningful: true
   # comments: Specifies the precision of "number" to be $MachinePrecision. Any magnitude must come after `."
 
+# How one determines when the ":" operator is a "Optional" or "Pattern" is a mystery.
+# But apparently they have difference precedences, and cannot be combined or thought
+# of as one operator.
 Optional:
   precedence: 140
   WolframLanguageData: 64
@@ -4466,10 +4469,17 @@ Optional:
   # N-tokens: {}
   # L-tokens: {":"}
   # O-tokens: {}
-  # usage: "patt : expr"
-  FullForm: Optional[patt, expr]
+  # usage: "symb : expr"
+  FullForm: Pattern[symb, expr]
   arity: Binary
   affix: Infix
+  # Should be right associative,
+  # but something in implicit multiplication
+  # is messing things up? In:
+  #   ConditionalExpression[expr1_, cond_]  expr2_
+  # ?
+  # Possibly this is due to a confusion between
+  # whether the ":" represents Optional or Pattern
   associativity: "unknown"
   meaningful: true
   # comments:
@@ -4718,6 +4728,9 @@ PartialUnderscriptBox:
   meaningful: false
   # comments: This operator is an invisible Unicode character and is used in the layout of displayed expressions.
 
+# How one determines when ":" is an operator is a Pattern or Optional, is a mystery.
+# But apparently they have difference precedences, and cannot be combined or thought
+# of as one operator.
 Pattern:
   precedence: 150
   WolframLanguageData: 64
@@ -5134,6 +5147,25 @@ RawBackquote:
   meaningful: true
   # comments: Specifies the accuracy (in number of digits to the right of decimal) of "number", where s is a positive decimal expressed without using ^^ or `. A magnitude must come after ``.
 
+# QuestionQuestion precedence needs to be less than Definition "?" which is
+# 5000 so that we don't treat ?? as ? ?.
+QuestionQuestion:
+  precedence: 5001  # Seems a bit extreme. This is the old Mathics data
+  WolframLanguageData: None
+  WolframLanguageData-corrected: None
+  UnicodeCharacters.tr: None
+  UnicodeCharacters-corrected.tr: None
+  # N-tokens: None
+  # L-tokens: None
+  # O-tokens: None
+  usage: "??Times"
+  FullForm: None
+  arity: Unary
+  affix: Prefix
+  associativity: null
+  meaningful: true
+  # comments: None
+
 Repeated:
   precedence: 170
   WolframLanguageData: 62