Handle non-printable ASCII in str and byte arrays (#169)

`"\xff"` or `'\xff'` are valid strings/char literals in C/C++. They were
not escaped and produced an invalid character that broke the
compilation.

After I escaped them, I realized that they cannot be translated to
`\xff` in Rust `str`'s because str only accepts `\x00 - \x7f`. Byte
array accepts `> \x7f`.

I changed Ptr::from_string_literal to accept a byte array instead of a
str. I also changed GetFmtArg to refuse to create Rust `str` that
contains chars `> \x7f` and fallback to GetRawArg that produces escaped
byte arrays.

Author: lucic71

cpp2rust/converter/converter.cpp

@@ -1332,7 +1332,11 @@ bool Converter::GetFmtArg(clang::Expr *arg, std::string &fmt,
                           std::string &fmt_args, const char *&fmt_trait,
                           std::string &fmt_width) {
   std::string arg_str = Mapper::ToString(arg);
-  if (clang::isa<clang::StringLiteral>(arg->IgnoreImplicit())) {
+  if (auto *str_lit =
+          clang::dyn_cast<clang::StringLiteral>(arg->IgnoreImplicit())) {
+    if (!IsAsciiStringLiteral(str_lit)) {
+      return false;
+    }
     auto str = GetEscapedStringLiteral(arg);
     std::string_view trim(str);
     // Delete " from string
@@ -1373,6 +1377,8 @@ bool Converter::GetRawArg(clang::Expr *arg, std::string &raw_args) {
     raw_args += "(&(" + str + ")[..(" + str + ").len() - 1]";
   } else if (Mapper::ToString(arg).contains("std::endl")) {
     raw_args += "(&[b'\\n']";
+  } else if (clang::isa<clang::StringLiteral>(arg->IgnoreImplicit())) {
+    raw_args += "(b" + GetEscapedStringLiteral(arg);
   } else {
     return false;
   }
@@ -1774,8 +1780,9 @@ bool Converter::VisitFloatingLiteral(clang::FloatingLiteral *expr) {
 }
 
 bool Converter::VisitCharacterLiteral(clang::CharacterLiteral *expr) {
+  auto uc = static_cast<unsigned char>(expr->getValue());
   std::string ch = GetEscapedCharLiteral(expr->getValue());
-  ch = "'" + std::move(ch) + "'";
+  ch = (uc > 0x7F ? "b'" : "'") + std::move(ch) + '\'';
   {
     PushParen paren(*this);
     StrCat(ch, keyword::kAs, ToStringBase(expr->getType()));
@@ -1802,7 +1809,7 @@ std::string Converter::GetEscapedCharLiteral(char character) const {
     return "\\0";
   }
   auto uc = static_cast<unsigned char>(character);
-  if (uc < 0x20 || uc == 0x7F) {
+  if (uc < 0x20 || uc >= 0x7F) {
     return std::format("\\x{:02x}", uc);
   }
   return std::string(1, character);

cpp2rust/converter/converter_lib.cpp

@@ -246,6 +246,15 @@ bool IsCallToOstream(clang::CallExpr *expr) {
   return false;
 }
 
+bool IsAsciiStringLiteral(const clang::StringLiteral *str) {
+  for (unsigned char c : str->getString()) {
+    if (c > 0x7F) {
+      return false;
+    }
+  }
+  return true;
+}
+
 std::vector<clang::CXXConstructorDecl *>
 GetTemplateInstantiatedCtors(clang::CXXRecordDecl *decl) {
   std::vector<clang::CXXConstructorDecl *> out;

cpp2rust/converter/converter_lib.h

@@ -65,6 +65,8 @@ bool IsUniquePtr(clang::QualType type);
 
 bool IsCallToOstream(clang::CallExpr *expr);
 
+bool IsAsciiStringLiteral(const clang::StringLiteral *str);
+
 std::vector<clang::CXXConstructorDecl *>
 GetTemplateInstantiatedCtors(clang::CXXRecordDecl *decl);
 

cpp2rust/converter/models/converter_refcount.cpp

@@ -1044,7 +1044,7 @@ bool ConverterRefCount::VisitStringLiteral(clang::StringLiteral *expr) {
                        GetEscapedStringLiteral(expr, pad)));
     return false;
   }
-  StrCat(GetEscapedStringLiteral(expr));
+  StrCat(std::format("b{}", GetEscapedStringLiteral(expr, 0)));
   return false;
 }
 

libcc2rs/src/rc.rs

@@ -940,9 +940,10 @@ impl fmt::Display for Ptr<u8> {
     }
 }
 
+type StringLiteralMap = HashMap<&'static [u8], Rc<RefCell<Vec<u8>>>>;
+
 thread_local! {
-    static STRING_LITERALS: RefCell<HashMap<&'static str, Rc<RefCell<Vec<u8>>>>> =
-        RefCell::new(HashMap::new());
+    static STRING_LITERALS: RefCell<StringLiteralMap> = RefCell::new(HashMap::new());
 }
 
 impl Ptr<u8> {
@@ -1020,12 +1021,12 @@ impl Ptr<u8> {
     }
 
     #[inline]
-    pub fn from_string_literal(s: &'static str) -> Self {
+    pub fn from_string_literal(s: &'static [u8]) -> Self {
         STRING_LITERALS.with(|literals| {
             let mut literals = literals.borrow_mut();
             let weak = Rc::downgrade(literals.entry(s).or_insert_with(|| {
                 Rc::new(RefCell::new({
-                    let mut v = s.as_bytes().to_vec();
+                    let mut v = s.to_vec();
                     v.push(0);
                     v
                 }))

tests/unit/out/refcount/bool_condition_logical.rs

@@ -127,7 +127,7 @@ fn main_0() -> i32 {
     if ((*n.borrow()) != 0) || (((*bits.borrow()) & 256_i64) != 0) {
         assert!(true);
     }
-    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::from_string_literal("hi")));
+    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::from_string_literal(b"hi")));
     let cnp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::<u8>::null()));
     if ((*x.borrow()) > (*y.borrow())) && (!(*cp.borrow()).is_null()) {
         assert!(true);

tests/unit/out/refcount/bool_condition_logical_c.rs

@@ -152,7 +152,7 @@ fn main_0() -> i32 {
     {
         assert!((1 != 0));
     }
-    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::from_string_literal("hi")));
+    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::from_string_literal(b"hi")));
     let cnp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::<u8>::null()));
     if (((((((*x.borrow()) > (*y.borrow())) as i32) != 0) && (!(*cp.borrow()).is_null())) as i32)
         != 0)

tests/unit/out/refcount/char_printing.rs

@@ -13,7 +13,7 @@ fn main_0() -> i32 {
     let vec_: Value<Vec<u8>> = Rc::new(RefCell::new(vec![195_u8, 167_u8]));
     let i: Value<i32> = Rc::new(RefCell::new(27));
     let str: Value<Vec<u8>> = Rc::new(RefCell::new(
-        Ptr::from_string_literal("rdas.")
+        Ptr::from_string_literal(b"rdas.")
             .to_c_string_iterator()
             .chain(std::iter::once(0))
             .collect::<Vec<u8>>(),
@@ -29,12 +29,16 @@ fn main_0() -> i32 {
         ]
         .concat()),
     );
-    write!(
-        libcc2rs::cout(),
-        "0x{:x} açordas?\nSim, 0x{:x}.\n",
-        27,
-        (*i.borrow()),
+    write!(libcc2rs::cout(), "0x{:x}", 27,);
+    libcc2rs::cout().write_all(
+        &([
+            (b" a\xc3\xa7ordas?" as &[u8]),
+            (&[('\n' as u8)] as &[u8]),
+            (b"Sim, 0x" as &[u8]),
+        ]
+        .concat()),
     );
+    write!(libcc2rs::cout(), "{:x}.\n", (*i.borrow()),);
     write!(libcc2rs::cout(), "Hello, World!\n",);
     libcc2rs::cout().write_all(
         &([

tests/unit/out/refcount/char_printing_cerr.rs

@@ -13,7 +13,7 @@ fn main_0() -> i32 {
     let vec_: Value<Vec<u8>> = Rc::new(RefCell::new(vec![195_u8, 167_u8]));
     let i: Value<i32> = Rc::new(RefCell::new(27));
     let str: Value<Vec<u8>> = Rc::new(RefCell::new(
-        Ptr::from_string_literal("rdas.")
+        Ptr::from_string_literal(b"rdas.")
             .to_c_string_iterator()
             .chain(std::iter::once(0))
             .collect::<Vec<u8>>(),
@@ -29,12 +29,16 @@ fn main_0() -> i32 {
         ]
         .concat()),
     );
-    write!(
-        libcc2rs::cerr(),
-        "0x{:x} açordas?\nSim, 0x{:x}.\n",
-        27,
-        (*i.borrow()),
+    write!(libcc2rs::cerr(), "0x{:x}", 27,);
+    libcc2rs::cerr().write_all(
+        &([
+            (b" a\xc3\xa7ordas?" as &[u8]),
+            (&[('\n' as u8)] as &[u8]),
+            (b"Sim, 0x" as &[u8]),
+        ]
+        .concat()),
     );
+    write!(libcc2rs::cerr(), "{:x}.\n", (*i.borrow()),);
     write!(libcc2rs::cerr(), "Hello, World!\n",);
     libcc2rs::cerr().write_all(
         &([

tests/unit/out/refcount/default_in_statics.rs

@@ -111,7 +111,7 @@ thread_local!(
 );
 thread_local!(
     pub static static_foo_3: Value<Foo> = Rc::new(RefCell::new(Foo {
-        s1: Rc::new(RefCell::new(Ptr::from_string_literal("hello"))),
+        s1: Rc::new(RefCell::new(Ptr::from_string_literal(b"hello"))),
         s2: Rc::new(RefCell::new(Ptr::<u8>::null())),
         fn1: Rc::new(RefCell::new(FnPtr::null())),
         fn2: Rc::new(RefCell::new(FnPtr::null())),
@@ -121,14 +121,14 @@ thread_local!(
 thread_local!(
     pub static static_foo_array_4: Value<Box<[Foo]>> = Rc::new(RefCell::new(Box::new([
         Foo {
-            s1: Rc::new(RefCell::new(Ptr::from_string_literal("first"))),
+            s1: Rc::new(RefCell::new(Ptr::from_string_literal(b"first"))),
             s2: Rc::new(RefCell::new(Ptr::<u8>::null())),
             fn1: Rc::new(RefCell::new(FnPtr::null())),
             fn2: Rc::new(RefCell::new(FnPtr::null())),
             n: Rc::new(RefCell::new(1)),
         },
         Foo {
-            s1: Rc::new(RefCell::new(Ptr::from_string_literal("second"))),
+            s1: Rc::new(RefCell::new(Ptr::from_string_literal(b"second"))),
             s2: Rc::new(RefCell::new(Ptr::<u8>::null())),
             fn1: Rc::new(RefCell::new(FnPtr::null())),
             fn2: Rc::new(RefCell::new(FnPtr::null())),