rustpython-literal

Author: youknowone

ast/Cargo.toml

@@ -11,10 +11,10 @@ license = "MIT"
 default = ["constant-optimization", "fold"]
 constant-optimization = ["fold"]
 fold = []
-unparse = ["rustpython-common"]
+unparse = ["rustpython-literal"]
 
 [dependencies]
 rustpython-compiler-core = { path = "../core", version = "0.2.0" }
-rustpython-common = { path = "../../common", version = "0.2.0", optional = true }
+rustpython-literal = { path = "../literal", version = "0.2.0", optional = true }
 
 num-bigint = { workspace = true }

ast/src/constant.rs

@@ -35,17 +35,17 @@ impl From<BigInt> for Constant {
     }
 }
 
-#[cfg(feature = "rustpython-common")]
+#[cfg(feature = "rustpython-literal")]
 impl std::fmt::Display for Constant {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Constant::None => f.pad("None"),
             Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
-            Constant::Str(s) => rustpython_common::escape::UnicodeEscape::new_repr(s.as_str())
+            Constant::Str(s) => rustpython_literal::escape::UnicodeEscape::new_repr(s.as_str())
                 .str_repr()
                 .write(f),
             Constant::Bytes(b) => {
-                let escape = rustpython_common::escape::AsciiEscape::new_repr(b);
+                let escape = rustpython_literal::escape::AsciiEscape::new_repr(b);
                 let repr = escape.bytes_repr().to_string().unwrap();
                 f.pad(&repr)
             }
@@ -64,7 +64,7 @@ impl std::fmt::Display for Constant {
                     f.write_str(")")
                 }
             }
-            Constant::Float(fp) => f.pad(&rustpython_common::float_ops::to_string(*fp)),
+            Constant::Float(fp) => f.pad(&rustpython_literal::float::to_string(*fp)),
             Constant::Complex { real, imag } => {
                 if *real == 0.0 {
                     write!(f, "{imag}j")

ast/src/unparse.rs

@@ -511,7 +511,7 @@ impl<'a> Unparser<'a> {
         } else {
             self.p("f")?;
             let body = to_string_fmt(|f| Unparser::new(f).unparse_fstring_body(values, is_spec));
-            rustpython_common::escape::UnicodeEscape::new_repr(&body)
+            rustpython_literal::escape::UnicodeEscape::new_repr(&body)
                 .str_repr()
                 .write(&mut self.f)
         }

literal/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "rustpython-literal"
+version = "0.2.0"
+description = "Common literal handling utilities mostly useful for unparse and repr."
+authors = ["RustPython Team"]
+edition = "2021"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+
+[dependencies]
+num-traits = { workspace = true }
+
+hexf-parse = "0.2.1"
+lexical-parse-float = { version = "0.8.0", features = ["format"] }
+unic-ucd-category = "0.9"
+
+[dev-dependencies]
+rand = { workspace = true }

literal/src/char.rs

@@ -0,0 +1,15 @@
+use unic_ucd_category::GeneralCategory;
+
+/// According to python following categories aren't printable:
+/// * Cc (Other, Control)
+/// * Cf (Other, Format)
+/// * Cs (Other, Surrogate)
+/// * Co (Other, Private Use)
+/// * Cn (Other, Not Assigned)
+/// * Zl Separator, Line ('\u2028', LINE SEPARATOR)
+/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR)
+/// * Zs (Separator, Space) other than ASCII space('\x20').
+pub fn is_printable(c: char) -> bool {
+    let cat = GeneralCategory::of(c);
+    !(cat.is_other() || cat.is_separator())
+}