experiment/tree sitter

.cargo/config.toml

@@ -4,5 +4,9 @@ test_details = ["test", "--target", "aarch64-apple-darwin"]
 [build]
 target = "wasm32-wasip1"
 
+[env]
+CC_wasm32_wasip1 = { value = "/Users/christian/.wasi-sdk/bin/clang", force = true }
+CFLAGS_wasm32_wasip1 = { value = "--sysroot=/Users/christian/.wasi-sdk/share/wasi-sysroot", force = true }
+
 [target.'cfg(all(target_arch = "wasm32"))']
 runner = "viceroy run -C ../../fastly.toml -- "

Cargo.toml

@@ -56,3 +56,6 @@ urlencoding = "2.1"
 uuid = { version = "1.18", features = ["v4"] }
 validator = { version = "0.20", features = ["derive"] }
 which = "8"
+tree-sitter = { version = "0.26.2" }
+tree-sitter-javascript = { version = "0.25.0" }
+

README.md

@@ -112,6 +112,26 @@ See https://asdf-vm.com/guide/getting-started.html#_2-configure-asdf
 git clone git@github.com:IABTechLab/trusted-server.git
 ```
 
+### WASI SDK (Required for Tree-sitter)
+
+This project uses tree-sitter for JavaScript parsing, which requires the WASI SDK to compile C code for WebAssembly.
+
+#### Download and Install WASI SDK
+
+```sh
+cd /tmp
+curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-arm64-macos.tar.gz
+tar -xzf wasi-sdk-24.0-arm64-macos.tar.gz
+mv wasi-sdk-24.0-arm64-macos ~/.wasi-sdk
+```
+
+:warning: For Linux or x86_64 macOS, download the appropriate release from [WASI SDK releases](https://github.com/WebAssembly/wasi-sdk/releases/tag/wasi-sdk-24):
+- Linux x86_64: `wasi-sdk-24.0-x86_64-linux.tar.gz`
+- Linux arm64: `wasi-sdk-24.0-arm64-linux.tar.gz`
+- macOS x86_64: `wasi-sdk-24.0-x86_64-macos.tar.gz`
+
+The WASI SDK paths are already configured in `.cargo/config.toml` and will be used automatically during build.
+
 ### Configure
 
 #### Edit configuration files

crates/common/Cargo.toml

@@ -45,6 +45,8 @@ uuid = { workspace = true }
 validator = { workspace = true }
 ed25519-dalek = { workspace = true }
 once_cell = { workspace = true }
+tree-sitter = { workspace = true }
+tree-sitter-javascript = { workspace = true }
 
 [build-dependencies]
 config = { workspace = true }

crates/common/src/lib.rs

@@ -43,4 +43,5 @@ pub mod streaming_processor;
 pub mod streaming_replacer;
 pub mod synthetic;
 pub mod test_support;
+pub mod tree_sitter_test;
 pub mod tsjs;

crates/common/src/tree_sitter_test.rs

@@ -0,0 +1,108 @@
+use tree_sitter::{Parser, Tree};
+
+/// Initialize a tree-sitter parser with JavaScript language support
+pub fn create_js_parser() -> Parser {
+    let mut parser = Parser::new();
+    let language = tree_sitter_javascript::LANGUAGE.into();
+    parser
+        .set_language(&language)
+        .expect("Failed to set JavaScript language");
+    parser
+}
+
+/// Parse JavaScript source code and return the syntax tree
+pub fn parse_js(source: &str) -> Option<Tree> {
+    let mut parser = create_js_parser();
+    parser.parse(source, None)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parser_creation() {
+        let parser = create_js_parser();
+        // Parser should be created successfully with JavaScript language
+        assert!(parser.language().is_some());
+    }
+
+    #[test]
+    fn test_parse_simple_function() {
+        let source = "function add(a, b) { return a + b; }";
+        let tree = parse_js(source).expect("Failed to parse JavaScript");
+
+        let root_node = tree.root_node();
+        assert_eq!(root_node.kind(), "program");
+        assert_eq!(root_node.child_count(), 1);
+
+        // First child should be a function declaration
+        let function_node = root_node.child(0).expect("Should have a child");
+        assert_eq!(function_node.kind(), "function_declaration");
+    }
+
+    #[test]
+    fn test_parse_variable_declaration() {
+        let source = "const x = 42;";
+        let tree = parse_js(source).expect("Failed to parse JavaScript");
+
+        let root_node = tree.root_node();
+        assert_eq!(root_node.kind(), "program");
+
+        // First child should be a lexical declaration
+        let declaration = root_node.child(0).expect("Should have a child");
+        assert_eq!(declaration.kind(), "lexical_declaration");
+    }
+
+    #[test]
+    fn test_parse_complex_code() {
+        let source = r#"
+            class Calculator {
+                constructor() {
+                    this.result = 0;
+                }
+
+                add(x, y) {
+                    return x + y;
+                }
+            }
+
+            const calc = new Calculator();
+            console.log(calc.add(5, 3));
+        "#;
+
+        let tree = parse_js(source).expect("Failed to parse JavaScript");
+        let root_node = tree.root_node();
+
+        assert_eq!(root_node.kind(), "program");
+        // Should have at least 3 children: class, const declaration, expression statement
+        assert!(root_node.child_count() >= 3);
+
+        // Verify the class declaration
+        let class_node = root_node.child(0).expect("Should have first child");
+        assert_eq!(class_node.kind(), "class_declaration");
+    }
+
+    #[test]
+    fn test_parse_arrow_function() {
+        let source = "const multiply = (a, b) => a * b;";
+        let tree = parse_js(source).expect("Failed to parse JavaScript");
+
+        let root_node = tree.root_node();
+        assert_eq!(root_node.kind(), "program");
+
+        let declaration = root_node.child(0).expect("Should have a child");
+        assert_eq!(declaration.kind(), "lexical_declaration");
+    }
+
+    #[test]
+    fn test_parse_with_syntax_error() {
+        // This should still produce a tree, but with error nodes
+        let source = "function broken( { return x; }";
+        let tree = parse_js(source);
+
+        assert!(tree.is_some());
+        let tree = tree.unwrap();
+        assert!(tree.root_node().has_error());
+    }
+}