From 65d77dd593926d8e59f252f20a01fef34ee747b5 Mon Sep 17 00:00:00 2001 From: Matthew Paras <34500476+mattwparas@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:26:41 -0800 Subject: [PATCH] Experimental defmacro changes (#126) * first pass for better defmacro support --- crates/steel-core/src/compiler/code_gen.rs | 2 +- crates/steel-core/src/compiler/compiler.rs | 101 +++++++++- crates/steel-core/src/compiler/modules.rs | 83 +++++++- .../steel-core/src/compiler/passes/mangle.rs | 1 + .../steel-core/src/parser/expand_visitor.rs | 177 ------------------ crates/steel-core/src/steel_vm/primitives.rs | 4 +- crates/steel-core/src/steel_vm/register_fn.rs | 126 +++++++++++++ crates/steel-language-server/src/backend.rs | 1 + 8 files changed, 301 insertions(+), 194 deletions(-) diff --git a/crates/steel-core/src/compiler/code_gen.rs b/crates/steel-core/src/compiler/code_gen.rs index a80d16fb3..9913596cb 100644 --- a/crates/steel-core/src/compiler/code_gen.rs +++ b/crates/steel-core/src/compiler/code_gen.rs @@ -542,7 +542,7 @@ impl<'a> VisitorMut for CodeGenerator<'a> { // directly from the stack self.push( LabeledInstruction::builder(OpCode::COPYCAPTURESTACK) - .payload(var.stack_offset.unwrap()) + .payload(var.stack_offset.ok_or_else(crate::throw!(Generic => format!("Error compiling this function - are you missing an expression after a local define?"); lambda_function.location.span))?) .contents(SyntaxObject::default(TokenType::Identifier(*key))), ); } diff --git a/crates/steel-core/src/compiler/compiler.rs b/crates/steel-core/src/compiler/compiler.rs index 9a9c5b380..912bac7b8 100644 --- a/crates/steel-core/src/compiler/compiler.rs +++ b/crates/steel-core/src/compiler/compiler.rs @@ -6,13 +6,13 @@ use crate::{ map::SymbolMap, passes::{ analysis::SemanticAnalysis, begin::flatten_begins_and_expand_defines, - reader::MultipleArityFunctions, shadow::RenameShadowedVariables, + reader::MultipleArityFunctions, shadow::RenameShadowedVariables, VisitorMutRefUnit, }, }, core::labels::Expr, parser::{ ast::AstTools, - expand_visitor::{expand_kernel, expand_kernel_in_env}, + expand_visitor::{expand_kernel, expand_kernel_in_env, expand_kernel_in_env_with_change}, interner::InternedString, kernel::Kernel, parser::{lower_entire_ast, lower_macro_and_require_definitions}, @@ -57,7 +57,7 @@ use crate::steel_vm::const_evaluation::ConstantEvaluatorManager; use super::{ constants::SerializableConstantMap, modules::{CompiledModule, ModuleManager}, - passes::analysis::Analysis, + passes::{analysis::Analysis, mangle::NameMangler}, program::RawProgramWithSymbols, }; @@ -278,6 +278,13 @@ pub enum OptLevel { Three, } +#[derive(Clone)] +pub(crate) struct KernelDefMacroSpec { + pub(crate) env: String, + pub(crate) exported: Option>, + pub(crate) name_mangler: NameMangler, +} + #[derive(Clone)] pub struct Compiler { pub(crate) symbol_map: SymbolMap, @@ -288,6 +295,12 @@ pub struct Compiler { pub(crate) kernel: Option, memoization_table: MemoizationTable, mangled_identifiers: HashSet, + // Try this out? + lifted_kernel_environments: HashMap, + // Macros that... we need to compile against directly at the top level + // This is really just a hack, but it solves cases for interactively + // running at the top level using private macros. + lifted_macro_environments: HashSet, } #[derive(Serialize, Deserialize)] @@ -352,6 +365,8 @@ impl Compiler { kernel: None, memoization_table: MemoizationTable::new(), mangled_identifiers: HashSet::new(), + lifted_kernel_environments: HashMap::new(), + lifted_macro_environments: HashSet::new(), } } @@ -371,6 +386,8 @@ impl Compiler { kernel: Some(kernel), memoization_table: MemoizationTable::new(), mangled_identifiers: HashSet::new(), + lifted_kernel_environments: HashMap::new(), + lifted_macro_environments: HashSet::new(), } } @@ -539,6 +556,8 @@ impl Compiler { exprs, path, builtin_modules, + &mut self.lifted_kernel_environments, + &mut self.lifted_macro_environments, ); #[cfg(not(feature = "modules"))] @@ -626,16 +645,46 @@ impl Compiler { expanded_statements = expanded_statements .into_iter() .map(|x| { + let mut x = x; + + for module in &self.lifted_macro_environments { + if let Some(macro_env) = self.modules().get(module).map(|x| &x.macro_map) { + x = crate::parser::expand_visitor::expand(x, macro_env)? + } + } + + // Lift all of the kernel macros as well? + for (module, lifted_env) in &mut self.lifted_kernel_environments { + let mut changed = false; + + (x, changed) = expand_kernel_in_env_with_change( + x, + self.kernel.as_mut(), + builtin_modules.clone(), + module.to_string(), + )?; + + if changed { + lifted_env.name_mangler.visit(&mut x); + } + } + expand_kernel_in_env( x, self.kernel.as_mut(), builtin_modules.clone(), "top-level".to_string(), ) + // TODO: If we have this, then we have to lower all of the expressions again .and_then(|x| crate::parser::expand_visitor::expand(x, &self.macro_env)) }) .collect::>>()?; + expanded_statements = expanded_statements + .into_iter() + .map(lower_entire_ast) + .collect::, ParseError>>()?; + log::debug!(target: "expansion-phase", "Beginning constant folding"); let mut expanded_statements = @@ -750,7 +799,22 @@ impl Compiler { expanded_statements = expanded_statements .into_iter() - .map(|x| { + .map(|mut x| { + // // Expanded any / all lifted environments: + // let mut x = x; + + // // Lift all of the kernel macros as well? + // for (module, lifted_env) in &self.lifted_kernel_environments { + // dbg!(&module); + + // x = expand_kernel_in_env( + // x, + // self.kernel.as_mut(), + // builtin_modules.clone(), + // module.to_string(), + // )?; + // } + expand_kernel_in_env( x, self.kernel.as_mut(), @@ -769,6 +833,30 @@ impl Compiler { expanded_statements = expanded_statements .into_iter() .map(|x| { + let mut x = x; + + for module in &self.lifted_macro_environments { + if let Some(macro_env) = self.modules().get(module).map(|x| &x.macro_map) { + x = crate::parser::expand_visitor::expand(x, macro_env)? + } + } + + // Lift all of the kernel macros as well? + for (module, lifted_env) in &mut self.lifted_kernel_environments { + let mut changed = false; + + (x, changed) = expand_kernel_in_env_with_change( + x, + self.kernel.as_mut(), + builtin_modules.clone(), + module.to_string(), + )?; + + if changed { + lifted_env.name_mangler.visit(&mut x); + } + } + expand_kernel_in_env( x, self.kernel.as_mut(), @@ -780,7 +868,10 @@ impl Compiler { }) .collect::>>()?; - // println!("{:#?}", expanded_statements); + expanded_statements = expanded_statements + .into_iter() + .map(lower_entire_ast) + .collect::, ParseError>>()?; // Let's do the @doc macro here? diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index cab29e422..bcc358c70 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -29,6 +29,7 @@ use crate::{ }; use crate::{parser::expand_visitor::Expander, rvals::Result}; +use once_cell::sync::Lazy; use steel_parser::expr_list; use std::{ @@ -50,6 +51,7 @@ use log::{debug, info, log_enabled}; use crate::parser::ast::IteratorExtensions; use super::{ + compiler::KernelDefMacroSpec, passes::{ begin::FlattenBegin, mangle::{collect_globals, NameMangler, NameUnMangler}, @@ -107,6 +109,8 @@ create_prelude!( for_syntax "#%private/steel/contract" ); +pub static STEEL_HOME: Lazy> = Lazy::new(|| std::env::var("STEEL_HOME").ok()); + /// Manages the modules /// keeps some visited state on the manager for traversal /// Also keeps track of the metadata for each file in order to determine @@ -195,6 +199,8 @@ impl ModuleManager { exprs: Vec, path: Option, builtin_modules: ModuleContainer, + lifted_kernel_environments: &mut HashMap, + lifted_macro_environments: &mut HashSet, ) -> Result> { // Wipe the visited set on entry self.visited.clear(); @@ -588,14 +594,47 @@ impl ModuleManager { // } if expander.changed || changed { - expand(first_round_expanded, &module.macro_map) + let fully_expanded = expand(first_round_expanded, &module.macro_map)?; + + let module_name = module.name.to_str().unwrap().to_string(); + + // Expanding the kernel with only these macros... + let (mut first_round_expanded, changed) = expand_kernel_in_env_with_change( + fully_expanded, + kernel.as_mut(), + // We don't need to expand those here + ModuleContainer::default(), + module_name.clone(), + // &kernel_macros_in_scope, + )?; + + if changed { + name_mangler.visit(&mut first_round_expanded); + } + + lifted_kernel_environments.insert( + module_name.clone(), + KernelDefMacroSpec { + env: module_name, + exported: None, + name_mangler: name_mangler.clone(), + }, + ); + + Ok(first_round_expanded) } else { Ok(first_round_expanded) } }) .collect::>()?; + // Global macro map - also need to expand with ALL macros + // post expansion in the target environment, which means we can't _just_ + // extend the global macro map with the target in scope macros, we need to + // do something like the two pass expansion global_macro_map.extend(in_scope_macros); + + lifted_macro_environments.insert(module.name.clone()); } // Include the defines from the modules now imported @@ -1762,7 +1801,36 @@ impl<'a> ModuleBuilder<'a> { // } if expander.changed || changed { - expand(first_round_expanded, &module.macro_map) + // expand(first_round_expanded, &module.macro_map) + + let fully_expanded = expand(first_round_expanded, &module.macro_map)?; + + let module_name = module.name.to_str().unwrap().to_string(); + + // Expanding the kernel with only these macros... + let (mut first_round_expanded, changed) = expand_kernel_in_env_with_change( + fully_expanded, + self.kernel.as_mut(), + // We don't need to expand those here + ModuleContainer::default(), + module_name.clone(), + // &kernel_macros_in_scope, + )?; + + if changed { + name_mangler.visit(&mut first_round_expanded); + } + + // lifted_kernel_environments.insert( + // module_name.clone(), + // KernelDefMacroSpec { + // env: module_name, + // exported: None, + // name_mangler: name_mangler.clone(), + // }, + // ); + + Ok(first_round_expanded) } else { Ok(first_round_expanded) } @@ -2192,13 +2260,10 @@ impl<'a> ModuleBuilder<'a> { let mut exprs_without_requires = Vec::new(); let exprs = std::mem::take(&mut self.source_ast); - let home = std::env::var("STEEL_HOME") - .map(PathBuf::from) - .map(|mut x| { - x.push("cogs"); - x - }) - .ok(); + let home = STEEL_HOME.clone().map(PathBuf::from).map(|mut x| { + x.push("cogs"); + x + }); fn walk( module_builder: &mut ModuleBuilder, diff --git a/crates/steel-core/src/compiler/passes/mangle.rs b/crates/steel-core/src/compiler/passes/mangle.rs index 59a18dd74..44b62eed9 100644 --- a/crates/steel-core/src/compiler/passes/mangle.rs +++ b/crates/steel-core/src/compiler/passes/mangle.rs @@ -128,6 +128,7 @@ impl<'a> VisitorMutRefUnit for NameUnMangler<'a> { } } +#[derive(Clone)] pub struct NameMangler { pub(crate) globals: HashSet, prefix: String, diff --git a/crates/steel-core/src/parser/expand_visitor.rs b/crates/steel-core/src/parser/expand_visitor.rs index 36e4d7d01..aef986ea1 100644 --- a/crates/steel-core/src/parser/expand_visitor.rs +++ b/crates/steel-core/src/parser/expand_visitor.rs @@ -670,183 +670,6 @@ impl<'a> ConsumingVisitor for KernelExpander<'a> { })) = l.first().cloned() { if let Some(map) = &mut self.map { - /* - - - if s == *DOC_MACRO { - if l.len() != 3 { - stop!(BadSyntax => "Malformed @doc statement!") - } - - let mut args = l.into_iter(); - args.next(); // Skip past the doc macro - - let comment = args.next().unwrap(); - let top_level_define = args.next().unwrap(); - - // println!("Getting here - {}", top_level_define); - // println!("{:?}", top_level_define); - - match &top_level_define { - // A classic @doc case - // (@doc "comment" (define )) - ExprKind::Define(d) => { - let doc_expr = ExprKind::Define(Box::new(Define::new( - ExprKind::atom( - d.name.atom_identifier().unwrap().to_string() + "__doc__", - ), - comment, - SyntaxObject::default(TokenType::Define), - ))); - - // let ast_name = ExprKind::atom( - // d.name.atom_identifier().unwrap().to_string() + "__ast__", - // ); - - // Include the metadata table - let metadata_table_addition = ExprKind::List(List::new(vec![ - ExprKind::atom("#%function-ptr-table-add"), - ExprKind::atom("#%function-ptr-table"), - ExprKind::atom(d.name.atom_identifier().unwrap().clone()), - ExprKind::atom( - d.name.atom_identifier().unwrap().to_string() + "__doc__", - ), - ])); - - let expanded_expr = self.visit(top_level_define)?; - - // let quoted_ast = define_quoted_ast_node(ast_name, &expanded_expr); - - return Ok(ExprKind::Begin(Begin::new( - vec![doc_expr, expanded_expr, metadata_table_addition], - SyntaxObject::default(TokenType::Begin), - ))); - } - - // An edge case that should eventually be realized: - // - ExprKind::Begin(b) => match &b.exprs.as_slice() { - // If this is a sequence of two things, catch the latter one like above - &[ExprKind::Define(d), ExprKind::Set(s), _] - if d.name.atom_identifier() == s.variable.atom_identifier() => - { - let doc_expr = ExprKind::Define(Box::new(Define::new( - ExprKind::atom( - d.name.atom_identifier().unwrap().to_string() + "__doc__", - ), - comment, - SyntaxObject::default(TokenType::Define), - ))); - - // let ast_name = ExprKind::atom( - // d.name.atom_identifier().unwrap().to_string() + "__ast__", - // ); - - let expanded_expr = self.visit(top_level_define)?; - - // let quoted_ast = define_quoted_ast_node(ast_name, &expanded_expr); - - return Ok(ExprKind::Begin(Begin::new( - vec![doc_expr, expanded_expr], - SyntaxObject::default(TokenType::Begin), - ))); - } - _ => return self.visit(top_level_define), - }, - - ExprKind::List(struct_def) - if struct_def.first_ident() == Some(&STRUCT_KEYWORD) => - { - if let Some(struct_name) = - struct_def.get(1).and_then(|x| x.atom_identifier()) - { - let doc_expr = ExprKind::Define(Box::new(Define::new( - ExprKind::atom(struct_name.to_string() + "__doc__"), - comment, - SyntaxObject::default(TokenType::Define), - ))); - - // let ast_name = ExprKind::atom(struct_name.to_string() + "__ast__"); - - // let quoted_ast = - // define_quoted_ast_node(ast_name, &top_level_define); - - return Ok(ExprKind::Begin(Begin::new( - vec![doc_expr, self.visit(top_level_define)?], - SyntaxObject::default(TokenType::Begin), - ))); - } - - return self.visit(top_level_define); - } - - // This should... probably happen after the expansion? - ExprKind::List(unlowered_define) - if unlowered_define.first_ident() == Some(&DEFINE) - || unlowered_define - .first() - .and_then(|x| x.atom_syntax_object()) - .map(|x| x.ty == TokenType::Define) - .unwrap_or_default() => - { - // Depends on how this has been expanded at this point. We could have - // (define (foo x) x) or (define foo (lambda (x) x)) - // So we need to handle accordingly, since the lowering hasn't happened - // yet. - - // println!("Matching define"); - - let name = match unlowered_define.get(1) { - Some(ExprKind::List(l)) => l.first_ident(), - Some(ExprKind::Atom(a)) => a.ident(), - _ => return self.visit(top_level_define), - } - .unwrap(); - - let doc_expr = ExprKind::Define(Box::new(Define::new( - ExprKind::atom(name.resolve().to_string() + "__doc__"), - comment, - SyntaxObject::default(TokenType::Define), - ))); - - // let ast_name = ExprKind::atom( - // d.name.atom_identifier().unwrap().to_string() + "__ast__", - // ); - - // Include the metadata table - let metadata_table_addition = ExprKind::List(List::new(vec![ - ExprKind::atom("#%function-ptr-table-add"), - ExprKind::atom("#%function-ptr-table"), - ExprKind::atom(*name), - ExprKind::atom(name.resolve().to_string() + "__doc__"), - ])); - - let expanded_expr = self.visit(top_level_define)?; - - // let quoted_ast = define_quoted_ast_node(ast_name, &expanded_expr); - - return Ok(ExprKind::Begin(Begin::new( - vec![doc_expr, expanded_expr, metadata_table_addition], - SyntaxObject::default(TokenType::Begin), - ))); - } - - ExprKind::List(unlowered_define) - if unlowered_define.first_ident() == Some(&BEGIN) => - { - return self.visit(top_level_define); - } - - _ => { - // println!("Not matching..."); - - return self.visit(top_level_define); - } - } - } - - */ - if map .contains_syntax_object_macro(&s, self.environment.as_ref().map(|x| x.as_ref())) && self diff --git a/crates/steel-core/src/steel_vm/primitives.rs b/crates/steel-core/src/steel_vm/primitives.rs index e9d8bd94c..b9ad7648a 100644 --- a/crates/steel-core/src/steel_vm/primitives.rs +++ b/crates/steel-core/src/steel_vm/primitives.rs @@ -1436,12 +1436,12 @@ fn make_mutable_box(ctx: &mut VmCore, args: &[SteelVal]) -> Option>) -> SteelVal { value.borrow().clone() } -#[steel_derive::function(name = "set-box!")] +#[steel_derive::function(name = "set-strong-box!")] pub fn set_box(value: &Gc>, update_to: SteelVal) { *value.borrow_mut() = update_to; } diff --git a/crates/steel-core/src/steel_vm/register_fn.rs b/crates/steel-core/src/steel_vm/register_fn.rs index 5b8734386..f37ec3198 100644 --- a/crates/steel-core/src/steel_vm/register_fn.rs +++ b/crates/steel-core/src/steel_vm/register_fn.rs @@ -523,6 +523,132 @@ impl< } } +impl< + RET: IntoSteelVal, + SELF: AsRefMutSteelValFromRef, + INNER: FromSteelVal + Clone + AsRefSteelValFromUnsized, + FN: Fn(&mut SELF, &[INNER]) -> RET + SendSyncStatic, + > RegisterFn, RET> for Engine +{ + fn register_fn(&mut self, name: &'static str, func: FN) -> &mut Self { + // use std::Borrow(); + + let f = move |args: &[SteelVal]| -> Result { + if args.len() != 2 { + stop!(ArityMismatch => format!("{} expected {} argument, got {}", name, 3, args.len())); + } + + let mut input = ::as_mut_ref_from_ref(&args[0])?; + + let temp_res = INNER::as_ref_from_unsized(&args[1])?; + + let res = func(&mut input, temp_res.as_slice_repr()); + + res.into_steelval() + }; + + self.register_value( + name, + SteelVal::BoxedFunction(Rc::new(BoxedDynFunction::new( + Arc::new(f), + Some(name), + Some(2), + ))), + ) + } + + fn register_owned_fn(&mut self, name: String, func: FN) -> &mut Self { + // use std::Borrow(); + + let cloned_name = name.clone(); + + let f = move |args: &[SteelVal]| -> Result { + if args.len() != 2 { + stop!(ArityMismatch => format!("{} expected {} argument, got {}", name, 3, args.len())); + } + + let mut input = ::as_mut_ref_from_ref(&args[0])?; + + let temp_res = INNER::as_ref_from_unsized(&args[1])?; + + let res = func(&mut input, temp_res.as_slice_repr()); + + res.into_steelval() + }; + + self.register_value( + &cloned_name.to_string(), + SteelVal::BoxedFunction(Rc::new(BoxedDynFunction::new_owned( + Arc::new(f), + Some(cloned_name.into()), + Some(2), + ))), + ) + } +} + +impl< + RET: IntoSteelVal, + SELF: AsRefMutSteelValFromRef, + INNER: FromSteelVal + Clone + AsRefSteelValFromUnsized, + FN: Fn(&mut SELF, &[INNER]) -> RET + SendSyncStatic, + > RegisterFn, RET> for BuiltInModule +{ + fn register_fn(&mut self, name: &'static str, func: FN) -> &mut Self { + // use std::Borrow(); + + let f = move |args: &[SteelVal]| -> Result { + if args.len() != 2 { + stop!(ArityMismatch => format!("{} expected {} argument, got {}", name, 3, args.len())); + } + + let mut input = ::as_mut_ref_from_ref(&args[0])?; + + let temp_res = INNER::as_ref_from_unsized(&args[1])?; + + let res = func(&mut input, temp_res.as_slice_repr()); + + res.into_steelval() + }; + + self.register_value( + name, + SteelVal::BoxedFunction(Rc::new(BoxedDynFunction::new( + Arc::new(f), + Some(name), + Some(2), + ))), + ) + } + + fn register_owned_fn(&mut self, name: String, func: FN) -> &mut Self { + let cloned_name = name.clone(); + + let f = move |args: &[SteelVal]| -> Result { + if args.len() != 2 { + stop!(ArityMismatch => format!("{} expected {} argument, got {}", name, 3, args.len())); + } + + let mut input = ::as_mut_ref_from_ref(&args[0])?; + + let temp_res = INNER::as_ref_from_unsized(&args[1])?; + + let res = func(&mut input, temp_res.as_slice_repr()); + + res.into_steelval() + }; + + self.register_value( + &cloned_name.to_string(), + SteelVal::BoxedFunction(Rc::new(BoxedDynFunction::new_owned( + Arc::new(f), + Some(cloned_name.into()), + Some(2), + ))), + ) + } +} + impl< // RET: IntoSteelVal, 'a, diff --git a/crates/steel-language-server/src/backend.rs b/crates/steel-language-server/src/backend.rs index 0cfca7c6b..74952a26d 100644 --- a/crates/steel-language-server/src/backend.rs +++ b/crates/steel-language-server/src/backend.rs @@ -642,6 +642,7 @@ fn filter_interned_string_with_char( && !resolved.starts_with("%") && !resolved.starts_with("mangler#%") && !resolved.starts_with("!!dummy-rest") + && !resolved.starts_with("__module-mangler") { Some(resolved.to_string()) } else {