Add parser deps to rustpython_ast_pyo3 (#75)

Author: youknowone

ast-pyo3/.gitignore

@@ -0,0 +1,72 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version

ast-pyo3/Cargo.toml

@@ -8,8 +8,13 @@ edition = "2021"
 # It reimplements AST types, but currently both slower than python AST types and limited to use in other API
 wrapper = []
 
+[lib]
+name = "rustpython_ast"
+crate-type = ["cdylib"]
+
 [dependencies]
 rustpython-ast = { workspace = true, features = ["location"] }
+rustpython-parser = { workspace = true }
 num-complex = { workspace = true }
 once_cell = { workspace = true }
 

ast-pyo3/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["maturin>=0.15,<0.16"]
+build-backend = "maturin"
+
+[project]
+name = "rustpython_ast"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+]
+
+
+[tool.maturin]
+features = ["pyo3/extension-module"]

ast-pyo3/src/lib.rs

@@ -3,3 +3,47 @@ mod py_ast;
 pub mod wrapper;
 
 pub use py_ast::{init, PyNode, ToPyAst};
+use pyo3::prelude::*;
+use rustpython_parser::ast::{source_code::SourceLocator, Fold};
+
+#[pyfunction]
+#[pyo3(signature = (source, filename="<unknown>", *, type_comments=false, locate=true))]
+pub fn parse<'py>(
+    source: &str,
+    filename: &str,
+    type_comments: bool,
+    locate: bool,
+    py: Python<'py>,
+) -> PyResult<&'py PyAny> {
+    if type_comments {
+        todo!("'type_comments' is not implemented yet");
+    }
+    let parsed = rustpython_parser::parse(source, rustpython_parser::Mode::Module, filename)
+        .map_err(|e| PyErr::new::<pyo3::exceptions::PySyntaxError, _>(e.to_string()))?;
+    if locate {
+        let parsed = SourceLocator::new(source).fold(parsed).unwrap();
+        parsed.module().unwrap().to_py_ast(py)
+    } else {
+        parsed.module().unwrap().to_py_ast(py)
+    }
+}
+
+#[pymodule]
+fn rustpython_ast(py: Python, m: &PyModule) -> PyResult<()> {
+    py_ast::init(py)?;
+
+    #[cfg(feature = "wrapper")]
+    {
+        let ast = PyModule::new(py, "ast")?;
+        wrapper::located::add_to_module(py, ast)?;
+        m.add_submodule(ast)?;
+
+        let ast = PyModule::new(py, "unlocated_ast")?;
+        wrapper::ranged::add_to_module(py, ast)?;
+        m.add_submodule(ast)?;
+    }
+
+    m.add_function(wrap_pyfunction!(parse, m)?)?;
+
+    Ok(())
+}

ast-pyo3/test_ast.py

@@ -0,0 +1,56 @@
+import re
+import difflib
+import pytest
+
+import ast as py_ast
+import rustpython_ast as rust_ast
+
+
+from glob import glob
+
+files = {}
+for path in glob("../../cpython/Lib/**/*.py"):
+    try:
+        txt = open(path, "r").read()
+    except UnicodeDecodeError:
+        continue
+
+    # try:
+    #     if py_ast.dump(py_ast.parse(txt)) != py_ast.dump(rust_ast.parse(txt)):
+    #         continue
+    # except SyntaxError:
+    #     continue
+    files[path] = txt
+
+
+@pytest.mark.parametrize("path", files.keys())
+def test_roundtrip(path):
+    txt = files[path]
+    module_p = py_ast.parse(txt)
+    dump_p = py_ast.dump(module_p, indent=True)
+    module_r = rust_ast.parse(txt)
+    dump_r = py_ast.dump(module_r, indent=True)
+    p = re.compile("object at 0x[0-9a-f]+")
+    dump_p2 = re.sub(p, "object at 0x????????", dump_p)
+    dump_r2 = re.sub(p, "object at 0x????????", dump_r)
+    try:
+        assert dump_p2 == dump_r2
+    except AssertionError:
+        last_sign = ' '
+        for s in difflib.ndiff(dump_p2, dump_r2):
+            if s[0]==' ': continue
+            if s[0] == last_sign:
+                print(s[2:], end='')
+            else:
+                print()
+                print(s, end='')
+            last_sign = s[0]
+        # with open("dump_code.py", "w") as f:
+        #     f.write(path)
+        #     f.write('\n')
+        #     f.write(txt)
+        # with open("dump_p.txt", "w") as f:
+        #     f.write(dump_p2)
+        # with open("dump_r.txt", "w") as f:
+        #     f.write(dump_r2)
+        raise