defmacro wip

crates/steel-core/src/compiler/modules.rs

@@ -9,7 +9,10 @@ use crate::{
     },
     parser::{
         ast::{AstTools, Atom, Begin, Define, ExprKind, List, Quote},
-        expand_visitor::{expand_kernel, expand_kernel_in_env},
+        expand_visitor::{
+            expand_kernel, expand_kernel_in_env, expand_kernel_in_env_with_allowed,
+            expand_kernel_in_env_with_change,
+        },
         interner::InternedString,
         kernel::Kernel,
         parser::{
@@ -18,7 +21,7 @@ use crate::{
         },
         tokens::TokenType,
     },
-    steel_vm::{engine::ModuleContainer, transducers::interleave},
+    steel_vm::{builtin::BuiltInModule, engine::ModuleContainer, transducers::interleave},
 };
 use crate::{parser::expand_visitor::Expander, rvals::Result};
 
@@ -490,33 +493,66 @@ impl ModuleManager {
             .filter(|x| x.for_syntax)
             .map(|x| x.path.get_path())
         {
-            let (module, mut in_scope_macros) = Self::find_in_scope_macros(
+            let (module, mut in_scope_macros, mut name_mangler) = Self::find_in_scope_macros(
                 &self.compiled_modules,
                 require_for_syntax.as_ref(),
                 &mut mangled_asts,
             );
 
-            // dbg!(&in_scope_macros);
-
-            // for (key, value) in &mut in_scope_macros {
-            //     for line in value.exprs_mut() {
-            //         println!("{}", line);
-            //     }
-            // }
-
-            // ast = ast.into_iter().map(|x| )
-
-            // ast.pretty_print();
+            let kernel_macros_in_scope: HashSet<_> =
+                module.provides_for_syntax.iter().cloned().collect();
 
             ast = ast
                 .into_iter()
                 .map(|x| {
+                    // @matt 12/8/2023
+                    // The easiest thing to do here, is to go to the other module, and find
+                    // what defmacros have been exposed on the require for syntax. Once those
+                    // have been found, we run a pass with kernel expansion, limiting the
+                    // expander to only use the macros that we've exposed. After that,
+                    // we run the expansion again, using the full suite of defmacro capabilities.
+                    //
+                    // The question that remains - how to define the neat phases of what kinds
+                    // of macros can expand into what? Can defmacro -> syntax-rules -> defmacro?
+                    // This could eventually prove to be cumbersome, but it is still early
+                    // for defmacro. Plus, I need to create a syntax-case or syntax-parse
+                    // frontend before the defmacro style macros become too pervasive.
+                    //
+                    // TODO: Replicate this behavior over to builtin modules
+
                     // First expand the in scope macros
                     // These are macros
                     let mut expander = Expander::new(&in_scope_macros);
-                    let first_round_expanded = expander.expand(x)?;
+                    let mut first_round_expanded = expander.expand(x)?;
+                    let mut changed = false;
+
+                    (first_round_expanded, changed) = expand_kernel_in_env_with_allowed(
+                        first_round_expanded,
+                        kernel.as_mut(),
+                        // We don't need to expand those here
+                        ModuleContainer::default(),
+                        module.name.to_str().unwrap().to_string(),
+                        &kernel_macros_in_scope,
+                    )?;
 
-                    if expander.changed {
+                    // If the kernel expander expanded into something - go ahead
+                    // and expand all of the macros in this
+                    if changed || expander.changed {
+                        // Expand here?
+                        first_round_expanded = expand(first_round_expanded, &module.macro_map)?;
+
+                        // Probably don't need this
+                        (first_round_expanded, changed) = expand_kernel_in_env_with_change(
+                            first_round_expanded,
+                            kernel.as_mut(),
+                            ModuleContainer::default(),
+                            module.name.to_str().unwrap().to_string(),
+                        )?;
+
+                        name_mangler.visit(&mut first_round_expanded);
+                    }
+
+                    if expander.changed || changed {
                         expand(first_round_expanded, &module.macro_map)
                     } else {
                         Ok(first_round_expanded)
@@ -553,7 +589,11 @@ impl ModuleManager {
         compiled_modules: &'a HashMap<PathBuf, CompiledModule>,
         require_for_syntax: &'a PathBuf,
         mangled_asts: &'a mut Vec<ExprKind>,
-    ) -> (&'a CompiledModule, HashMap<InternedString, SteelMacro>) {
+    ) -> (
+        &'a CompiledModule,
+        HashMap<InternedString, SteelMacro>,
+        NameMangler,
+    ) {
         let module = compiled_modules
             .get(require_for_syntax)
             .expect(&format!("Module missing!: {:?}", require_for_syntax));
@@ -595,11 +635,11 @@ impl ModuleManager {
             })
             .collect::<HashMap<_, _>>();
         // Check what macros are in scope here
-        debug!(
-            "In scope macros: {:#?}",
-            in_scope_macros.keys().collect::<Vec<_>>()
-        );
-        (module, in_scope_macros)
+        // println!(
+        //     "In scope macros: {:#?}",
+        //     in_scope_macros.keys().collect::<Vec<_>>()
+        // );
+        (module, in_scope_macros, name_mangler)
     }
 
     #[cfg(not(feature = "modules"))]
@@ -1492,7 +1532,7 @@ impl<'a> ModuleBuilder<'a> {
             .filter(|x| x.for_syntax)
             .map(|x| x.path.get_path())
         {
-            let (module, in_scope_macros) = ModuleManager::find_in_scope_macros(
+            let (module, in_scope_macros, name_mangler) = ModuleManager::find_in_scope_macros(
                 self.compiled_modules,
                 require_for_syntax.as_ref(),
                 &mut mangled_asts,
@@ -1501,10 +1541,16 @@ impl<'a> ModuleBuilder<'a> {
             ast = ast
                 .into_iter()
                 .map(|x| {
+                    for (key, _) in &in_scope_macros {
+                        println!("1513 in scope macro Macro found: {}", key);
+                    }
                     // First expand the in scope macros
                     // These are macros
                     let mut expander = Expander::new(&in_scope_macros);
                     let first_round_expanded = expander.expand(x)?;
+                    for (key, _) in &module.macro_map {
+                        println!("1520 Macro found: {}", key);
+                    }
 
                     if expander.changed {
                         expand(first_round_expanded, &module.macro_map)

crates/steel-core/src/parser/expand_visitor.rs

@@ -26,7 +26,7 @@ use super::{
 };
 
 use std::borrow::Cow;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use crate::parser::expander::SteelMacro;
 
@@ -209,6 +209,43 @@ impl<'a> ConsumingVisitor for Expander<'a> {
     }
 }
 
+pub fn expand_kernel_in_env_with_allowed(
+    expr: ExprKind,
+    kernel: Option<&mut Kernel>,
+    builtin_modules: ModuleContainer,
+    env: String,
+    allowed: &HashSet<InternedString>,
+) -> Result<(ExprKind, bool)> {
+    let mut expander = KernelExpander {
+        map: kernel,
+        changed: false,
+        builtin_modules,
+        environment: Some(Cow::from(env)),
+        depth: 0,
+        allowed_macros: Some(allowed),
+    };
+
+    expander.visit(expr).map(|x| (x, expander.changed))
+}
+
+pub fn expand_kernel_in_env_with_change(
+    expr: ExprKind,
+    kernel: Option<&mut Kernel>,
+    builtin_modules: ModuleContainer,
+    env: String,
+) -> Result<(ExprKind, bool)> {
+    let mut expander = KernelExpander {
+        map: kernel,
+        changed: false,
+        builtin_modules,
+        environment: Some(Cow::from(env)),
+        depth: 0,
+        allowed_macros: None,
+    };
+
+    expander.visit(expr).map(|x| (x, expander.changed))
+}
+
 pub fn expand_kernel_in_env(
     expr: ExprKind,
     kernel: Option<&mut Kernel>,
@@ -221,6 +258,7 @@ pub fn expand_kernel_in_env(
         builtin_modules,
         environment: Some(Cow::from(env)),
         depth: 0,
+        allowed_macros: None,
     }
     .visit(expr)
 }
@@ -236,6 +274,7 @@ pub fn expand_kernel(
         builtin_modules,
         environment: None,
         depth: 0,
+        allowed_macros: None,
     }
     .visit(expr)
 }
@@ -246,6 +285,7 @@ pub struct KernelExpander<'a> {
     builtin_modules: ModuleContainer,
     environment: Option<Cow<'static, str>>,
     depth: usize,
+    allowed_macros: Option<&'a HashSet<InternedString>>,
 }
 
 impl<'a> KernelExpander<'a> {
@@ -256,6 +296,7 @@ impl<'a> KernelExpander<'a> {
             builtin_modules,
             environment: None,
             depth: 0,
+            allowed_macros: None,
         }
     }
 
@@ -714,6 +755,11 @@ impl<'a> ConsumingVisitor for KernelExpander<'a> {
 
                 if map
                     .contains_syntax_object_macro(&s, self.environment.as_ref().map(|x| x.as_ref()))
+                    && self
+                        .allowed_macros
+                        .as_ref()
+                        .map(|x| x.contains(&s))
+                        .unwrap_or(true)
                 {
                     let expanded = map.expand_syntax_object(
                         &s,

crates/steel-core/src/primitives/lists.rs

@@ -638,7 +638,8 @@ mod list_operation_tests {
     fn cons_test_normal_input() {
         let mut args = [SteelVal::IntV(1), SteelVal::IntV(2)];
         let res = cons(&mut args);
-        let expected = SteelVal::ListV(vec![SteelVal::IntV(1), SteelVal::IntV(2)].into());
+
+        let expected = SteelVal::Pair(Gc::new(Pair::cons(SteelVal::IntV(1), SteelVal::IntV(2))));
 
         assert_eq!(res.unwrap(), expected);
     }

crates/steel-parser/src/lexer.rs

@@ -74,8 +74,6 @@ impl<'a> Lexer<'a> {
         let mut buf = String::new();
         while let Some(&c) = self.chars.peek() {
             self.eat();
-            // println!("{}", c);
-            // println!("{:?}", buf);
             match c {
                 '"' => return Ok(TokenType::StringLiteral(buf)),
                 '\\' => match self.chars.peek() {
@@ -104,6 +102,11 @@ impl<'a> Lexer<'a> {
                         buf.push('\r');
                     }
 
+                    Some('0') => {
+                        self.eat();
+                        buf.push('\0');
+                    }
+
                     _ => return Err(TokenError::InvalidEscape),
                 },
                 _ => buf.push(c),
@@ -399,8 +402,6 @@ impl<'a> Iterator for Lexer<'a> {
     type Item = Result<TokenType<&'a str>>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        // self.consume_whitespace_and_comments_until_next_input();
-
         // Crunch until the next input
         self.consume_whitespace();
 
@@ -779,6 +780,13 @@ mod lexer_tests {
         println!("{:#?}", res);
     }
 
+    #[test]
+    fn lex_string_with_escape_chars() {
+        let s = TokenStream::new("\"\0\0\0\"", true, None);
+        let res: Vec<Token<&str>> = s.collect();
+        println!("{:#?}", res);
+    }
+
     #[test]
     fn scheme_statement() {
         let s = TokenStream::new("(apples (function a b) (+ a b))", true, None);

crates/steel-parser/src/parser.rs

@@ -267,7 +267,7 @@ pub type Result<T> = result::Result<T, ParseError>;
 
 fn tokentype_error_to_parse_error(t: &Token<'_, InternedString>) -> ParseError {
     if let TokenType::Error = t.ty {
-        // println!("Found an error: {}", t);
+        println!("Found an error: {}", t.typ());
 
         if t.source.starts_with('\"') {
             ParseError::IncompleteString(t.source.to_string(), t.span, None)