v2: add parser

Cargo.toml

@@ -28,21 +28,28 @@ simplelog = "0.12.0"
 structopt = "0.3"
 tempfile = "3.2.0"
 toml = "0.5.9"
+dir-test = "0.4"
+drop_bomb = "0.1.5"
+camino = "1.1.9"
+pg_query = "6.1.0"
 
 # local
 squawk-parser = { version = "0.0.0", path = "./crates/parser" }
 squawk-linter = { version = "0.0.0", path = "./crates/linter" }
 squawk-github = { version = "0.0.0", path = "./crates/github" }
+squawk_lexer = { version = "0.0.0", path = "./crates/squawk_lexer" }
 
 [workspace.lints.clippy]
 collapsible_else_if = "allow"
 collapsible_if = "allow"
 needless_return = "allow"
 doc_markdown = "deny"
 manual_let_else = "deny"
+explicit_iter_loop = "deny"
 
 [profile.dev]
 debug = 0
 
 [profile.dev.package]
 insta.opt-level = 3
+similar.opt-level = 3

crates/squawk_lexer/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "lexer"
+name = "squawk_lexer"
 version = "0.0.0"
 description = "TBD"
 

crates/squawk_parser/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "squawk_parser"
+version = "0.0.0"
+description = "TBD"
+
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+
+[lib]
+doctest = false
+
+[dependencies]
+squawk_lexer.workspace = true
+drop_bomb.workspace = true
+
+[dev-dependencies]
+insta.workspace = true
+dir-test.workspace = true
+camino.workspace = true
+pg_query.workspace = true
+
+[lints]
+workspace = true

crates/squawk_parser/src/event.rs

@@ -0,0 +1,172 @@
+// via https://github.com/rust-lang/rust-analyzer/blob/d8887c0758bbd2d5f752d5bd405d4491e90e7ed6/crates/parser/src/event.rs
+//
+// Permission is hereby granted, free of charge, to any
+// person obtaining a copy of this software and associated
+// documentation files (the "Software"), to deal in the
+// Software without restriction, including without
+// limitation the rights to use, copy, modify, merge,
+// publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software
+// is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice
+// shall be included in all copies or substantial portions
+// of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+//! This module provides a way to construct a `File`.
+//! It is intended to be completely decoupled from the
+//! parser, so as to allow to evolve the tree representation
+//! and the parser algorithm independently.
+
+use std::mem;
+
+use crate::{output::Output, syntax_kind::SyntaxKind};
+
+/// `Parser` produces a flat list of `Event`s.
+/// They are converted to a tree-structure in
+/// a separate pass, via `SyntaxTreeBuilder`.
+#[derive(Debug)]
+pub(crate) enum Event {
+    /// This event signifies the start of the node.
+    /// It should be either abandoned (in which case the
+    /// `kind` is `TOMBSTONE`, and the event is ignored),
+    /// or completed via a `Finish` event.
+    ///
+    /// All tokens between a `Start` and a `Finish` would
+    /// become the children of the respective node.
+    ///
+    /// For left-recursive syntactic constructs, the parser produces
+    /// a child node before it sees a parent. `forward_parent`
+    /// saves the position of current event's parent.
+    ///
+    /// Consider this path
+    ///
+    /// `foo::bar`
+    ///
+    /// The events for it would look like this:
+    ///
+    /// ```text
+    /// START(PATH) IDENT('foo') FINISH START(PATH) T![::] IDENT('bar') FINISH
+    ///       |                          /\
+    ///       |                          |
+    ///       +------forward-parent------+
+    /// ```
+    ///
+    /// And the tree would look like this
+    ///
+    /// ```text
+    ///    +--PATH---------+
+    ///    |   |           |
+    ///    |   |           |
+    ///    |  '::'       'bar'
+    ///    |
+    ///   PATH
+    ///    |
+    ///   'foo'
+    /// ```
+    ///
+    /// See also `CompletedMarker::precede`.
+    Start {
+        kind: SyntaxKind,
+        forward_parent: Option<u32>,
+    },
+
+    /// Complete the previous `Start` event
+    Finish,
+
+    /// Produce a single leaf-element.
+    /// `n_raw_tokens` is used to glue complex contextual tokens.
+    /// For example, lexer tokenizes `>>` as `>`, `>`, and
+    /// `n_raw_tokens = 2` is used to produced a single `>>`.
+    Token {
+        kind: SyntaxKind,
+        n_raw_tokens: u8,
+    },
+    /// When we parse `foo.0.0` or `foo. 0. 0` the lexer will hand us a float literal
+    /// instead of an integer literal followed by a dot as the lexer has no contextual knowledge.
+    /// This event instructs whatever consumes the events to split the float literal into
+    /// the corresponding parts.
+    FloatSplitHack {
+        ends_in_dot: bool,
+    },
+    Error {
+        msg: String,
+    },
+}
+
+impl Event {
+    pub(crate) fn tombstone() -> Self {
+        Event::Start {
+            kind: SyntaxKind::TOMBSTONE,
+            forward_parent: None,
+        }
+    }
+}
+
+/// Generate the syntax tree with the control of events.
+pub(super) fn process(mut events: Vec<Event>) -> Output {
+    let mut res = Output::default();
+    let mut forward_parents = Vec::new();
+
+    for i in 0..events.len() {
+        match mem::replace(&mut events[i], Event::tombstone()) {
+            Event::Start {
+                kind,
+                forward_parent,
+            } => {
+                // For events[A, B, C], B is A's forward_parent, C is B's forward_parent,
+                // in the normal control flow, the parent-child relation: `A -> B -> C`,
+                // while with the magic forward_parent, it writes: `C <- B <- A`.
+
+                // append `A` into parents.
+                forward_parents.push(kind);
+                let mut idx = i;
+                let mut fp = forward_parent;
+                while let Some(fwd) = fp {
+                    idx += fwd as usize;
+                    // append `A`'s forward_parent `B`
+                    fp = match mem::replace(&mut events[idx], Event::tombstone()) {
+                        Event::Start {
+                            kind,
+                            forward_parent,
+                        } => {
+                            forward_parents.push(kind);
+                            forward_parent
+                        }
+                        _ => unreachable!(),
+                    };
+                    // append `B`'s forward_parent `C` in the next stage.
+                }
+
+                for kind in forward_parents.drain(..).rev() {
+                    if kind != SyntaxKind::TOMBSTONE {
+                        res.enter_node(kind);
+                    }
+                }
+            }
+            Event::Finish => res.leave_node(),
+            Event::Token { kind, n_raw_tokens } => {
+                res.token(kind, n_raw_tokens);
+            }
+            Event::FloatSplitHack { ends_in_dot } => {
+                res.float_split_hack(ends_in_dot);
+                let ev = mem::replace(&mut events[i + 1], Event::tombstone());
+                assert!(matches!(ev, Event::Finish), "{ev:?}");
+            }
+            Event::Error { msg } => res.error(msg),
+        }
+    }
+
+    res
+}