update tests and fix bugs in parser (#64)

* skip trivial first tokens in parsing

* remove comment in input test

* manage out-of-bound case, update input test

* update input test

* update input test

* make test programs private

* remove comments in syntax tests

* format

* make Pragma optional, remove ROOT

* rename close() params, use advance() in eat(), fix typo scope

* parse params using list_identity

* return Option in token_value() and position_of()

* do not allow <--, <== in var declaration

* make public signal optional in main component

* fix format before merge

* remove empty test

* replace expect by eat in block

* add scope parsing test

* fix space in scope parsing test

* update snapshot test for syntax

* refactor syntax test

* update input tests with snapshot

* change param type in find signal into str

* add snapshot guild

* add some combine arithmetic operators

* error report in expect and expect_any

* update snapshot for combine operators

* update snapshot test

* fix wrap trivial tokens before open a tree

* update template happy test

* add operators into token kinds

* add operators test

* re-priority token kinds

* fix clippy check

* fix clippy check

* rebase add-token-kinds

* fix clippy check

* fix clippy

* remove duplicate declaration in grammar tests

* remove duplicate else in comment

* replace eat by expect for curly in block

.gitignore

@@ -3,4 +3,6 @@ node_modules
 .vscode-test
 /target
 Cargo.lock
-assets
+assets
+*.new
+*.pending-snap

Cargo.toml

@@ -10,5 +10,6 @@ syntax = {path = './crates/syntax', version = "0.1.0"}
 circom-lsp = {path = './crates/lsp', version = "*"}
 common = { path = './crates/common', version = "*"}
 database = {path = "./crates/database", version = "*"}
+
 [workspace.package]
 rust-version = "1.71"

SNAPSHOT_TEST.md

@@ -0,0 +1,11 @@
+# Snapshot Test
+
+* Run all tests:
+    ```
+    cargo test
+    ```
+
+* Review snapshot changes
+    ```
+    cargo insta review
+    ```

crates/parser/Cargo.toml

@@ -11,7 +11,16 @@ lsp-types = {version = "0.94.1", features = ["proposed"]}
 rowan = "0.15.15"
 num-traits = "0.2"
 num-derive = "0.2"
-
+serde = "1.0.216"
 
 [profile.dev]
-debug = 2
+debug = 2
+
+[dev-dependencies]
+# for snapshot testing, yaml format
+insta = { version = "1.41.1", features = ["yaml"] }
+
+[profile.dev.package]
+# compile slightly slower once, but use less memory, have faster diffs
+insta.opt-level = 3
+similar.opt-level = 3

crates/parser/src/event.rs

@@ -1,8 +1,9 @@
 use crate::token_kind::TokenKind;
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone)]
 pub enum Event {
     Open { kind: TokenKind },
     Close,
     TokenPosition(usize),
+    ErrorReport(String),
 }

crates/parser/src/grammar/block.rs

@@ -1,13 +1,21 @@
 use super::*;
 
+/*
+{
+    <declaration>/<statement>
+    <declaration>/<statement>
+    ....
+    <declaration>/<statement>
+}
+*/
 pub fn block(p: &mut Parser) {
     p.inc_rcurly();
 
     if !p.at(LCurly) {
         p.advance_with_error("Miss {");
     } else {
         let m = p.open();
-        p.eat(LCurly);
+        p.expect(LCurly);
         let stmt_marker = p.open();
         while !p.at(RCurly) && !p.eof() {
             let kind = p.current();

crates/parser/src/grammar/declaration.rs

@@ -128,9 +128,3 @@ pub(super) fn declaration(p: &mut Parser) {
         _ => unreachable!(),
     }
 }
-
-#[cfg(test)]
-mod declar_tests {
-    #[test]
-    fn signal_with_tag() {}
-}

crates/parser/src/grammar/expression.rs

@@ -156,25 +156,3 @@ fn circom_expression(p: &mut Parser) {
         }
     }
 }
-// #[cfg(test)]
-// mod tests {
-
-//     use rowan::SyntaxNode;
-
-//     use crate::{syntax_node::CircomLang};
-
-//     use super::{entry::Scope, Parser};
-
-//     #[test]
-//     fn test_expression() {
-//         let source = r#"
-// {
-//           a.tmp <== 100;
-//           b[1].c <== 10;
-//     }
-//         "#;
-//         let green = Parser::parse_scope(source, Scope::Block);
-//         let node = SyntaxNode::<CircomLang>::new_root(green);
-//         println!("{:#?}", node);
-//     }
-// }

crates/parser/src/grammar/pragma.rs

@@ -14,26 +14,3 @@ pub fn pragma(p: &mut Parser) {
     p.expect(Semicolon);
     p.close(m, Pragma);
 }
-
-// #[cfg(test)]
-// mod tests {
-//     #[test]
-//     fn pragam_test() {
-//         use crate::{
-//             ast::{AstNode, AstPragma},
-//             syntax_node::SyntaxNode,
-//             token_kind::TokenKind,
-//         };
-
-//         use super::{entry::Scope, Parser};
-
-//         let source: String = r#"pragma circom 2.0.1;"#.to_string();
-
-//         let green_node = Parser::parse_scope(&source, Scope::Pragma);
-//         let node = SyntaxNode::new_root(green_node);
-
-//         let pragma = AstPragma::cast(node.last_child().unwrap()).unwrap();
-
-//         assert!(pragma.version().unwrap().syntax().kind() == TokenKind::Version);
-//     }
-// }

crates/parser/src/grammar/statement.rs

@@ -9,6 +9,12 @@ pub(super) fn statement(p: &mut Parser) {
     p.close(m, Statement);
 }
 
+/*
+if (expr)
+    <statement>
+else
+    <statement>
+*/
 fn if_statement(p: &mut Parser) {
     let m = p.open();
     p.expect(IfKw);
@@ -25,6 +31,7 @@ fn if_statement(p: &mut Parser) {
 
 /**
  * no if condition here.
+ * for/while/return/assert...
  */
 fn statement_no_condition(p: &mut Parser) {
     match p.current() {
@@ -50,6 +57,10 @@ fn statement_no_condition(p: &mut Parser) {
     }
 }
 
+/*
+for (<declaration>/<assignment>; <expression>; <assignment>)
+    <statement>
+*/
 fn for_statement(p: &mut Parser) {
     let m = p.open();
     p.expect(ForKw);
@@ -70,6 +81,10 @@ fn for_statement(p: &mut Parser) {
     p.close(m, ForLoop);
 }
 
+/*
+while (<expression>)
+    <statement>
+*/
 fn while_statement(p: &mut Parser) {
     p.expect(WhileKw);
     p.expect(LParen);
@@ -78,6 +93,9 @@ fn while_statement(p: &mut Parser) {
     statement(p);
 }
 
+/*
+assert(<expression>)
+*/
 fn assert_statement(p: &mut Parser) {
     let m = p.open();
     p.expect(AssertKw);
@@ -87,6 +105,9 @@ fn assert_statement(p: &mut Parser) {
     p.close(m, AssertKw);
 }
 
+/*
+log()
+*/
 fn log_statement(p: &mut Parser) {
     let m = p.open();
     p.expect(LogKw);
@@ -109,13 +130,19 @@ fn log_statement(p: &mut Parser) {
     p.close(m, LogKw);
 }
 
+/*
+return <expression>
+*/
 fn return_statement(p: &mut Parser) {
     let m = p.open();
     p.expect(ReturnKw);
     expression(p);
     p.close(m, ReturnKw);
 }
 
+/*
+
+*/
 fn assignment_statement(p: &mut Parser) {
     let m = p.open();
 
@@ -155,20 +182,3 @@ fn assignment_statement(p: &mut Parser) {
         p.close(m, Error);
     }
 }
-
-#[cfg(test)]
-mod tests {
-
-    #[test]
-    fn if_statement_test() {
-        let _source = r#"
-            assert(1 == 2);
-        "#;
-        // let mut parser = Parser::new(source);
-
-        // statement(&mut parser);
-        // let cst = parser.build_tree().ok().unwrap();
-
-        // println!("{:?}", cst);
-    }
-}

crates/parser/src/grammar/template.rs

@@ -23,42 +23,3 @@ pub fn template(p: &mut Parser) {
 
     p.close(m, TemplateDef);
 }
-
-// #[cfg(test)]
-// mod tests {
-//     use crate::ast::AstTemplateDef;
-
-//     #[test]
-//     fn template_parse_test() {
-//         use crate::{ast::AstNode, syntax_node::SyntaxNode};
-
-//         use super::{entry::Scope, Parser};
-
-//         let source: String = r#"
-//         template Multiplier2 (a, b, c) {
-
-//            // Declaration of signals.
-//            signal input a;
-//            signal input b;
-//            signal output c;
-
-//            // Constraints.
-//            c <== a * b;
-//         }
-
-//         "#
-//         .to_string();
-
-//         let green_node = ::parse_scope(&source, Scope::Template);
-//         let node = SyntaxNode::new_root(green_node);
-
-//         let ast_template = AstTemplateDef::cast(node);
-
-//         if let Some(ast_internal) = ast_template {
-//             println!(
-//                 "name {:?}",
-//                 ast_internal.template_name().unwrap().syntax().text()
-//             );
-//         }
-//     }
-// }