Parse Trait (#71)

youknowone · web-flow · commit 4de0cb18277f · 2023-05-28T21:03:27.000+09:00
diff --git a/parser/README.md b/parser/README.md
@@ -60,7 +60,8 @@ Execution mode check. Allowed modes are `exec`, `eval` or `single`.
 
 For example, one could do this:
 ```
-  use rustpython_parser::{parser, ast};
+  use rustpython_parser::{Parse, ast};
   let python_source = "print('Hello world')";
-  let python_ast = parser::parse_expression(python_source).unwrap();
+  let python_statements = ast::Suite::parse(python_source).unwrap();  // statements
+  let python_expr = ast::Expr::parse(python_source).unwrap();  // or expr
 ```
diff --git a/parser/src/context.rs b/parser/src/context.rs
@@ -49,135 +49,135 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
 
 #[cfg(test)]
 mod tests {
-    use crate::parser::parse_program;
+    use crate::{ast, Parse};
 
     #[test]
     fn test_assign_name() {
         let source = "x = (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_tuple() {
         let source = "(x, y) = (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     #[cfg(not(feature = "all-nodes-with-ranges"))]
     fn test_assign_list() {
         let source = "[x, y] = (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_attribute() {
         let source = "x.y = (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_subscript() {
         let source = "x[y] = (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_starred() {
         let source = "(x, *y) = (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_for() {
         let source = "for x in (1, 2, 3): pass";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     #[cfg(not(feature = "all-nodes-with-ranges"))]
     fn test_assign_list_comp() {
         let source = "x = [y for y in (1, 2, 3)]";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     #[cfg(not(feature = "all-nodes-with-ranges"))]
     fn test_assign_set_comp() {
         let source = "x = {y for y in (1, 2, 3)}";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     #[cfg(not(feature = "all-nodes-with-ranges"))]
     fn test_assign_with() {
         let source = "with 1 as x: pass";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_named_expr() {
         let source = "if x:= 1: pass";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_ann_assign_name() {
         let source = "x: int = 1";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_aug_assign_name() {
         let source = "x += 1";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_aug_assign_attribute() {
         let source = "x.y += (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_aug_assign_subscript() {
         let source = "x[y] += (1, 2, 3)";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_del_name() {
         let source = "del x";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_del_attribute() {
         let source = "del x.y";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_del_subscript() {
         let source = "del x[y]";
-        let parse_ast = parse_program(source, "<test>").unwrap();
+        let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }
diff --git a/parser/src/function.rs b/parser/src/function.rs
@@ -155,15 +155,15 @@ const fn is_starred(exp: &ast::Expr) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::parser::{parse_program, ParseErrorType};
+    use crate::{ast, parser::ParseErrorType, Parse};
 
     #[cfg(not(feature = "all-nodes-with-ranges"))]
     macro_rules! function_and_lambda {
         ($($name:ident: $code:expr,)*) => {
             $(
                 #[test]
                 fn $name() {
-                    let parse_ast = parse_program($code, "<test>");
+                    let parse_ast = ast::Suite::parse($code, "<test>");
                     insta::assert_debug_snapshot!(parse_ast);
                 }
             )*
@@ -190,7 +190,7 @@ mod tests {
     }
 
     fn function_parse_error(src: &str) -> LexicalErrorType {
-        let parse_ast = parse_program(src, "<test>");
+        let parse_ast = ast::Suite::parse(src, "<test>");
         parse_ast
             .map_err(|e| match e.error {
                 ParseErrorType::Lexical(e) => e,
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
@@ -212,7 +212,7 @@ pub fn lex_starts_at(
     source: &str,
     mode: Mode,
     start_offset: TextSize,
-) -> impl Iterator<Item = LexResult> + '_ {
+) -> SoftKeywordTransformer<Lexer<std::str::Chars<'_>>> {
     SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode)
 }
 
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
@@ -94,13 +94,13 @@
 //! mode or tokenizing the source beforehand:
 //!
 //! ```
-//! use rustpython_parser::parse_program;
+//! use rustpython_parser::{Parse, ast};
 //!
 //! let python_source = r#"
 //! def is_odd(i):
 //!   return bool(i & 1)
 //! "#;
-//! let ast = parse_program(python_source, "<embedded>");
+//! let ast = ast::Suite::parse(python_source, "<embedded>");
 //!
 //! assert!(ast.is_ok());
 //! ```
@@ -126,13 +126,13 @@ mod soft_keywords;
 mod string;
 mod token;
 
-pub use parser::{
-    parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at,
-    parse_tokens, ParseError, ParseErrorType,
-};
+pub use parser::{parse, parse_starts_at, parse_tokens, Parse, ParseError, ParseErrorType};
 pub use string::FStringErrorType;
 pub use token::{StringKind, Tok};
 
+#[allow(deprecated)]
+pub use parser::{parse_expression, parse_expression_starts_at, parse_program};
+
 #[rustfmt::skip]
 mod python {
     #![allow(clippy::all)]
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
diff --git a/parser/src/string.rs b/parser/src/string.rs

Original file line number	Diff line number	Diff line change
`@@ -212,7 +212,7 @@ pub fn lex_starts_at(`
`212`	`212`	`source: &str,`
`213`	`213`	`mode: Mode,`
`214`	`214`	`start_offset: TextSize,`
`215`		`-) -> impl Iterator<Item = LexResult> + '_ {`
	`215`	`+) -> SoftKeywordTransformer<Lexer<std::str::Chars<'_>>> {`
`216`	`216`	`SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode)`
`217`	`217`	`}`
`218`	`218`