From 9f75fff385bed8692be08bd6e6d4581a8c256412 Mon Sep 17 00:00:00 2001 From: mattwparas Date: Thu, 7 Dec 2023 22:59:07 -0800 Subject: [PATCH 01/16] basic improper list implementation added --- .../steel-core/src/compiler/passes/shadow.rs | 4 ++ crates/steel-core/src/parser/ast.rs | 3 + crates/steel-core/src/parser/expander.rs | 16 +---- crates/steel-core/src/parser/kernel.rs | 2 + crates/steel-core/src/primitives/lists.rs | 58 +++++++++++++------ crates/steel-core/src/primitives/strings.rs | 14 +++++ crates/steel-core/src/rvals.rs | 10 ++++ crates/steel-core/src/rvals/cycles.rs | 43 ++++++++++++++ crates/steel-core/src/scheme/stdlib.scm | 3 - .../src/steel_vm/const_evaluation.rs | 55 ++++-------------- crates/steel-core/src/steel_vm/primitives.rs | 8 +++ crates/steel-core/src/values/closed.rs | 5 ++ crates/steel-core/src/values/lists.rs | 22 +++++++ 13 files changed, 162 insertions(+), 81 deletions(-) diff --git a/crates/steel-core/src/compiler/passes/shadow.rs b/crates/steel-core/src/compiler/passes/shadow.rs index e059c6c1c..63da1f338 100644 --- a/crates/steel-core/src/compiler/passes/shadow.rs +++ b/crates/steel-core/src/compiler/passes/shadow.rs @@ -82,6 +82,8 @@ impl VisitorMutRefUnit for RenameShadowedVariables { mut_var.push_str(self.str_modifiers.get(&modifier).unwrap()); } + // println!("Mangling variable: {}", mut_var); + *variable = mut_var.into(); self.scope.define(*variable); @@ -98,6 +100,8 @@ impl VisitorMutRefUnit for RenameShadowedVariables { self.shadows.pop_layer(); } + fn visit_quote(&mut self, _quote: &mut steel_parser::ast::Quote) {} + fn visit_atom(&mut self, a: &mut Atom) { if let Some(ident) = a.ident_mut() { if let Some(modifier) = self.shadows.get(ident) { diff --git a/crates/steel-core/src/parser/ast.rs b/crates/steel-core/src/parser/ast.rs index d2435c5fe..9bc19bc54 100644 --- a/crates/steel-core/src/parser/ast.rs +++ b/crates/steel-core/src/parser/ast.rs @@ -224,6 +224,9 @@ impl TryFrom<&SteelVal> for ExprKind { CharV(x) => Ok(ExprKind::Atom(Atom::new(SyntaxObject::default( CharacterLiteral(*x), )))), + + Pair(_) => Err("Can't convert from pair to expression!"), + // StructClosureV(_) => Err("Can't convert from struct-function to expression!"), PortV(_) => Err("Can't convert from port to expression!"), Closure(_) => Err("Can't convert from bytecode closure to expression"), diff --git a/crates/steel-core/src/parser/expander.rs b/crates/steel-core/src/parser/expander.rs index e62accd0e..ad7557d61 100644 --- a/crates/steel-core/src/parser/expander.rs +++ b/crates/steel-core/src/parser/expander.rs @@ -192,7 +192,7 @@ impl SteelMacro { .into_iter() .map(|x| { x.atom_identifier_or_else( - throw!(BadSyntax => "macros only support identifiers for special syntax"; sp), + throw!(BadSyntax => format!("macros only support identifiers for special syntax, found: {}", x); sp), ) .cloned() }) @@ -217,28 +217,14 @@ impl SteelMacro { // I think it should also not be greedy, and should report if there are ambiguous matchings fn match_case(&self, expr: &List) -> Result<&MacroCase> { for case in &self.cases { - // dbg!(&case); - - // dbg!(case.has_ellipses()) - if (case.has_ellipses() && expr.len() >= (case.arity() - 1)) || case.arity() == expr.len() { - // dbg!(case); - if case.recursive_match(expr) { return Ok(case); - } else { - // println!("Recursive match failed: case didn't match {:?}", case.args); } - } else { - // println!("Case didn't match: {:?}", case.args); - // dbg!(case.has_ellipses()); - // dbg!(case.arity()); - // dbg!(expr.len()); } } - error!("Macro expansion unable to match case with: {}", expr); if let Some(ExprKind::Atom(a)) = expr.first() { diff --git a/crates/steel-core/src/parser/kernel.rs b/crates/steel-core/src/parser/kernel.rs index f8110ac81..1a174231a 100644 --- a/crates/steel-core/src/parser/kernel.rs +++ b/crates/steel-core/src/parser/kernel.rs @@ -507,6 +507,8 @@ impl Kernel { expr: ExprKind, environment: &str, ) -> Result { + // println!("Expanding: {} with {}", ident, expr); + let span = get_span(&expr); let syntax_objects = diff --git a/crates/steel-core/src/primitives/lists.rs b/crates/steel-core/src/primitives/lists.rs index de8186b6e..61e70fb7c 100644 --- a/crates/steel-core/src/primitives/lists.rs +++ b/crates/steel-core/src/primitives/lists.rs @@ -1,6 +1,10 @@ -use crate::steel_vm::{ - builtin::{BuiltInModule, DocTemplate}, - vm::{apply, VmContext, APPLY_DOC}, +use crate::{ + gc::Gc, + steel_vm::{ + builtin::{BuiltInModule, DocTemplate}, + vm::{apply, VmContext, APPLY_DOC}, + }, + values::lists::Pair, }; use crate::{ rvals::{IntoSteelVal, Result, SteelVal}, @@ -221,9 +225,11 @@ fn is_empty(list: &SteelVal) -> bool { /// ``` #[steel_derive::function(name = "pair?")] fn pair(list: &SteelVal) -> bool { - list.list() - .map(|x| x.iter().next().is_some()) - .unwrap_or_default() + match list { + SteelVal::ListV(l) => l.iter().next().is_some(), + SteelVal::Pair(_) => true, + _ => false, + } } pub(crate) const CONS_DOC: DocTemplate<'static> = DocTemplate { @@ -246,7 +252,11 @@ pub fn cons(args: &mut [SteelVal]) -> Result { // Consider moving in a default value instead of cloning? Ok(SteelVal::ListV(right.clone())) } - (left, right) => Ok(SteelVal::ListV(vec![left, right.clone()].into())), + // Silly, but this then gives us a special "pair" that is different + // from a real bonafide list + (left, right) => Ok(SteelVal::Pair(Gc::new(Pair::cons(left, right.clone())))), + // TODO: Replace with an immutable pair here! + // (left, right) => Ok(SteelVal::ListV(vec![left, right.clone()].into())), } } @@ -363,9 +373,16 @@ fn first(list: &List) -> Result { /// > (car (cons 2 3)) ;; => 2 /// ``` #[steel_derive::function(name = "car", constant = true)] -fn car(list: &List) -> Result { - list.car() - .ok_or_else(throw!(Generic => "car resulted in an error - empty list")) +fn car(list: &SteelVal) -> Result { + match list { + SteelVal::ListV(l) => l + .car() + .ok_or_else(throw!(Generic => "car resulted in an error - empty list")), + + SteelVal::Pair(p) => Ok(p.car()), + + _ => stop!(TypeMismatch => "car expected a list or pair, found: {}", list), + } } pub(crate) const CDR_DOC: DocTemplate<'static> = DocTemplate { @@ -390,17 +407,22 @@ pub(crate) const CDR_DOC: DocTemplate<'static> = DocTemplate { fn cdr(args: &mut [SteelVal]) -> Result { arity_check!(rest, args, 1); - if let SteelVal::ListV(l) = &mut args[0] { - if l.is_empty() { - stop!(Generic => "cdr expects a non empty list"); + match &mut args[0] { + SteelVal::ListV(l) => { + if l.is_empty() { + stop!(Generic => "cdr expects a non empty list"); + } + + match l.rest_mut() { + Some(l) => Ok(SteelVal::ListV(l.clone())), + None => Ok(SteelVal::ListV(l.clone())), + } } - match l.rest_mut() { - Some(l) => Ok(SteelVal::ListV(l.clone())), - None => Ok(SteelVal::ListV(l.clone())), + SteelVal::Pair(p) => Ok(p.cdr()), + _ => { + stop!(TypeMismatch => format!("cdr expects a list, found: {}", &args[0])) } - } else { - stop!(TypeMismatch => format!("cdr expects a list, found: {}", &args[0])) } } diff --git a/crates/steel-core/src/primitives/strings.rs b/crates/steel-core/src/primitives/strings.rs index 099d7e0c6..aa79d17af 100644 --- a/crates/steel-core/src/primitives/strings.rs +++ b/crates/steel-core/src/primitives/strings.rs @@ -130,6 +130,20 @@ fn string_to_number_impl(value: &str, radix: Option) -> Result { } } +#[function(name = "make-string")] +pub fn make_string(k: usize, mut c: RestArgsIter<'_, char>) -> Result { + // If the char is there, we want to take it + let char = c.next(); + + // We want the iterator to be exhaused + if let Some(next) = c.next() { + stop!(ArityMismatch => format!("make-string expected 1 or 2 arguments, got an additional argument {}", next?)) + } + + let c = char.unwrap_or(Ok('\0'))?; + Ok((0..k).into_iter().map(|_| c).collect::().into()) +} + #[function(name = "string->number", constant = true)] pub fn string_to_number( value: &SteelString, diff --git a/crates/steel-core/src/rvals.rs b/crates/steel-core/src/rvals.rs index d3c3f2520..dce4530e9 100644 --- a/crates/steel-core/src/rvals.rs +++ b/crates/steel-core/src/rvals.rs @@ -1016,6 +1016,9 @@ pub enum SteelVal { // CompiledFunction(Box), // List ListV(crate::values::lists::List), + + Pair(Gc), + // Mutable functions MutFunc(MutFunctionSignature), // Built in functions @@ -1545,6 +1548,13 @@ impl SteelVal { } } + pub fn pair(&self) -> Option<&Gc> { + match self { + Self::Pair(p) => Some(p), + _ => None, + } + } + pub fn bool_or_else E>(&self, err: F) -> std::result::Result { match self { Self::BoolV(v) => Ok(*v), diff --git a/crates/steel-core/src/rvals/cycles.rs b/crates/steel-core/src/rvals/cycles.rs index 84bf4ce8f..64d096407 100644 --- a/crates/steel-core/src/rvals/cycles.rs +++ b/crates/steel-core/src/rvals/cycles.rs @@ -1,3 +1,4 @@ +use crate::values::lists::Pair; use std::{cell::Cell, collections::VecDeque}; use num::BigInt; @@ -166,6 +167,9 @@ impl CycleDetector { write!(f, "#\\{c}") } } + Pair(p) => { + write!(f, "({} . {})", p.car(), p.cdr()) + } FuncV(func) => { if let Some(name) = get_function_name(*func) { write!(f, "#", name.name) @@ -291,6 +295,9 @@ impl CycleDetector { write!(f, "#") } } + Pair(p) => { + write!(f, "({} . {})", p.car(), p.cdr()) + } Void => write!(f, "#"), SymbolV(s) => write!(f, "{s}"), VectorV(lst) => { @@ -698,6 +705,12 @@ impl<'a> BreadthFirstSearchSteelValVisitor for CycleCollector<'a> { self.push_back(heap_ref.get()); } } + + // TODO: Revisit this! + fn visit_pair(&mut self, pair: Gc) -> Self::Output { + self.push_back(pair.car()); + self.push_back(pair.cdr()); + } } #[cfg(not(feature = "without-drop-protection"))] @@ -1113,6 +1126,7 @@ impl<'a> BreadthFirstSearchSteelValVisitor for IterativeDropHandler<'a> { Reference(r) => self.visit_reference_value(r), BigNum(b) => self.visit_bignum(b), HeapAllocated(b) => self.visit_heap_allocated(b), + Pair(p) => self.visit_pair(p), }; } @@ -1120,6 +1134,13 @@ impl<'a> BreadthFirstSearchSteelValVisitor for IterativeDropHandler<'a> { ret } + + fn visit_pair(&mut self, pair: Gc) -> Self::Output { + if let Ok(inner) = Gc::try_unwrap(pair) { + self.push_back(inner.car); + self.push_back(inner.cdr); + } + } } pub trait BreadthFirstSearchSteelValVisitor { @@ -1168,6 +1189,7 @@ pub trait BreadthFirstSearchSteelValVisitor { Reference(r) => self.visit_reference_value(r), BigNum(b) => self.visit_bignum(b), HeapAllocated(b) => self.visit_heap_allocated(b), + Pair(p) => self.visit_pair(p), }; } @@ -1206,6 +1228,7 @@ pub trait BreadthFirstSearchSteelValVisitor { fn visit_reference_value(&mut self, reference: Rc>) -> Self::Output; fn visit_bignum(&mut self, bignum: Gc) -> Self::Output; fn visit_heap_allocated(&mut self, heap_ref: HeapRef) -> Self::Output; + fn visit_pair(&mut self, pair: Gc) -> Self::Output; } pub trait BreadthFirstSearchSteelValReferenceVisitor<'a> { @@ -1254,6 +1277,7 @@ pub trait BreadthFirstSearchSteelValReferenceVisitor<'a> { Reference(r) => self.visit_reference_value(r), BigNum(b) => self.visit_bignum(b), HeapAllocated(b) => self.visit_heap_allocated(b), + Pair(p) => self.visit_pair(p), }; } @@ -1298,6 +1322,7 @@ pub trait BreadthFirstSearchSteelValReferenceVisitor<'a> { ) -> Self::Output; fn visit_bignum(&mut self, bignum: &'a Gc) -> Self::Output; fn visit_heap_allocated(&mut self, heap_ref: &'a HeapRef) -> Self::Output; + fn visit_pair(&mut self, pair: &'a Gc) -> Self::Output; } thread_local! { @@ -1413,6 +1438,19 @@ impl<'a> RecursiveEqualityHandler<'a> { continue; } + // If we run into issues with any stack overflow + // check here first. + (Pair(l), Pair(r)) => { + if Gc::ptr_eq(&l, &r) { + continue; + } + + self.left.push_back(l.car()); + self.right.push_back(r.car()); + + self.left.push_back(l.cdr()); + self.right.push_back(r.cdr()); + } (BoolV(l), BoolV(r)) => { if l != r { return false; @@ -1834,6 +1872,11 @@ impl<'a> BreadthFirstSearchSteelValVisitor for EqualityVisitor<'a> { self.push_back(heap_ref.get()); // } } + + fn visit_pair(&mut self, pair: Gc) -> Self::Output { + self.push_back(pair.car()); + self.push_back(pair.cdr()); + } } impl PartialEq for SteelVal { diff --git a/crates/steel-core/src/scheme/stdlib.scm b/crates/steel-core/src/scheme/stdlib.scm index a75914a38..04e3ddecf 100644 --- a/crates/steel-core/src/scheme/stdlib.scm +++ b/crates/steel-core/src/scheme/stdlib.scm @@ -421,9 +421,6 @@ (define curry2 (lambda (func arg1) (lambda (arg2 arg3) (func arg1 arg2 arg3)))) ; (define compose (lambda (f g) (lambda (arg) (f (g arg))))) -(define (not a) - (if a #f #t)) - (define (foldl func accum lst) (if (null? lst) accum diff --git a/crates/steel-core/src/steel_vm/const_evaluation.rs b/crates/steel-core/src/steel_vm/const_evaluation.rs index 8d3c9ccbe..a9546d9f0 100644 --- a/crates/steel-core/src/steel_vm/const_evaluation.rs +++ b/crates/steel-core/src/steel_vm/const_evaluation.rs @@ -320,63 +320,22 @@ impl<'a> ConstantEvaluator<'a> { if evaluated_func.is_function() { match evaluated_func { SteelVal::MutFunc(f) => { - // println!( - // "Evaluating function!: {} with args: {:?}", - // func.atom_identifier().unwrap().resolve(), - // args - // ); - - // TODO: Clean this up - we shouldn't even enter this section of the code w/o having - // the actual atom itself. - // let output = if let Some(output) = self - // .memoization_table - // .get(SteelVal::FuncV(f), args.to_vec()) - // { - // // println!("Function result found in the cache!"); - // // println!("{:#?}", self.memoization_table); - - // output - // } else { - // println!("Not found in the cache, adding..."); - // println!("{:#?}", self.memoization_table); - let mut args = args.to_vec(); let output = f(&mut args) .map_err(|e| e.set_span_if_none(func.atom_syntax_object().unwrap().span))?; - // self.memoization_table.insert( - // SteelVal::FuncV(f), - // args.to_vec(), - // output.clone(), - // ); - - // output - // }; - self.handle_output(output, func, &evaluated_func, raw_args) } SteelVal::FuncV(f) => { - // println!( - // "Evaluating function!: {} with args: {:?}", - // func.atom_identifier().unwrap().resolve(), - // args - // ); - // TODO: Clean this up - we shouldn't even enter this section of the code w/o having // the actual atom itself. let output = if let Some(output) = self .memoization_table .get(SteelVal::FuncV(f), args.to_vec()) { - // println!("Function result found in the cache!"); - // println!("{:#?}", self.memoization_table); - output } else { - // println!("Not found in the cache, adding..."); - // println!("{:#?}", self.memoization_table); - let output = f(args).map_err(|e| { e.set_span_if_none(func.atom_syntax_object().unwrap().span) })?; @@ -621,10 +580,16 @@ impl<'a> ConsumingVisitor for ConstantEvaluator<'a> { // let span = get_span(&func_expr); if let Some(evaluated_func) = self.to_constant(&func_expr) { - debug!( - "Attempting to evaluate: {} with args: {:?}", - &func_expr, arguments - ); + // println!( + // "Attempting to evaluate: {} with args: {:?}", + // &func_expr, arguments + // ); + // TODO: This shouldn't fail here under normal circumstances! If the end result is an error, we should + // just return the value that was originally passed in. Otherwise, this signals + // an error in the dataflow, and it means we're checking a condition that isn't constant + // before applying a check against a constant value (which probably means we're missing) + // something in the constant evaluation check. In which case, we should probably + // just not stop the execution just because we errored return self.eval_function(evaluated_func, func_expr, args, &arguments); } else if let Some(ident) = func_expr.atom_identifier().and_then(|x| { // TODO: @Matt 4/24/23 - this condition is super ugly and I would prefer if we cleaned it up diff --git a/crates/steel-core/src/steel_vm/primitives.rs b/crates/steel-core/src/steel_vm/primitives.rs index 4964099d8..4ac37cdc2 100644 --- a/crates/steel-core/src/steel_vm/primitives.rs +++ b/crates/steel-core/src/steel_vm/primitives.rs @@ -273,6 +273,8 @@ pub const CONSTANTS: &[&str] = &[ "#%prim.list->string", LIST, PRIM_LIST, + "#%prim.not", + "not", ]; thread_local! { @@ -625,6 +627,11 @@ fn vector_module() -> BuiltInModule { module } +#[steel_derive::function(name = "not", constant = true)] +fn not(value: &SteelVal) -> bool { + matches!(value, SteelVal::BoolV(false)) +} + #[steel_derive::function(name = "int?", constant = true)] fn intp(value: &SteelVal) -> bool { matches!(value, SteelVal::IntV(_)) @@ -743,6 +750,7 @@ fn identity_module() -> BuiltInModule { let mut module = BuiltInModule::new("steel/identity"); module // .register_value("int?", gen_pred!(IntV)) + .register_native_fn_definition(NOT_DEFINITION) .register_native_fn_definition(INTEGERP_DEFINITION) .register_native_fn_definition(INTP_DEFINITION) .register_native_fn_definition(FLOATP_DEFINITION) diff --git a/crates/steel-core/src/values/closed.rs b/crates/steel-core/src/values/closed.rs index cc52337c2..59060dc68 100644 --- a/crates/steel-core/src/values/closed.rs +++ b/crates/steel-core/src/values/closed.rs @@ -788,4 +788,9 @@ impl<'a> BreadthFirstSearchSteelValVisitor for MarkAndSweepContext<'a> { } fn visit_void(&mut self) -> Self::Output {} + + fn visit_pair(&mut self, pair: Gc) -> Self::Output { + self.push_back(pair.car()); + self.push_back(pair.cdr()); + } } diff --git a/crates/steel-core/src/values/lists.rs b/crates/steel-core/src/values/lists.rs index ee809d933..ca0a0362b 100644 --- a/crates/steel-core/src/values/lists.rs +++ b/crates/steel-core/src/values/lists.rs @@ -4,6 +4,28 @@ use im_lists::handler::DropHandler; use crate::SteelVal; +// TODO: +// Builtin immutable pairs +#[derive(Clone)] +pub struct Pair { + pub(crate) car: SteelVal, + pub(crate) cdr: SteelVal, +} + +impl Pair { + pub fn cons(car: SteelVal, cdr: SteelVal) -> Self { + Pair { car, cdr } + } + + pub fn car(&self) -> SteelVal { + self.car.clone() + } + + pub fn cdr(&self) -> SteelVal { + self.cdr.clone() + } +} + #[cfg(feature = "without-drop-protection")] type DropHandlerChoice = im_lists::handler::DefaultDropHandler; #[cfg(not(feature = "without-drop-protection"))] From 237d4fe41fdd932a66a87b33bda439528f12e1da Mon Sep 17 00:00:00 2001 From: mattwparas Date: Fri, 8 Dec 2023 08:56:57 -0800 Subject: [PATCH 02/16] add some more functions --- crates/steel-core/src/primitives/lists.rs | 35 ++++++++++++++++++++++- crates/steel-core/src/rvals.rs | 3 ++ crates/steel-core/src/values/lists.rs | 6 ++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/steel-core/src/primitives/lists.rs b/crates/steel-core/src/primitives/lists.rs index 61e70fb7c..c63de373b 100644 --- a/crates/steel-core/src/primitives/lists.rs +++ b/crates/steel-core/src/primitives/lists.rs @@ -103,7 +103,8 @@ pub fn list_module() -> BuiltInModule { .register_value("transduce", crate::steel_vm::transducers::TRANSDUCE) .register_native_fn_definition(SECOND_DEFINITION) .register_native_fn_definition(THIRD_DEFINITION) - .register_native_fn_definition(TAKE_DEFINITION); + .register_native_fn_definition(TAKE_DEFINITION) + .register_native_fn_definition(LIST_TAIL_DEFINITION); module } @@ -176,6 +177,38 @@ fn _test_map(ctx: &mut VmCore, args: &[SteelVal]) -> Result { } } +#[steel_derive::function(name = "list-tail")] +pub fn list_tail(list_or_pair: &SteelVal, pos: usize) -> Result { + match list_or_pair { + SteelVal::ListV(l) => l + .tail(pos) + .ok_or_else(throw!(Generic => format!("list-tail expects at least {} + elements in the list, found: {}", pos, l.len()))) + .map(SteelVal::ListV), + SteelVal::Pair(p) => { + let mut value = p.cdr(); + let mut count = 1; + + while count < pos { + count += 1; + value = value + .pair() + .map(|x| x.cdr()) + .ok_or_else(throw!(Generic => format!("list-tail: index reached a + non-pair: index: {} in {}", count, list_or_pair)))?; + } + + Ok(value) + } + + _ if pos == 0 => Ok(list_or_pair.clone()), + + _ => { + stop!(TypeMismatch => format!("list-tail expects either a list or a pair, found: {}", list_or_pair)) + } + } +} + /// Returns a newly allocated list containing the vs as its elements. /// /// (list v ...) -> list? diff --git a/crates/steel-core/src/rvals.rs b/crates/steel-core/src/rvals.rs index dce4530e9..50812e503 100644 --- a/crates/steel-core/src/rvals.rs +++ b/crates/steel-core/src/rvals.rs @@ -1256,6 +1256,9 @@ impl std::fmt::Debug for SteelString { } } +// Check that steel values aren't growing without us knowing +const _ASSERT_SMALL: () = assert!(std::mem::size_of::() <= 16); + #[test] fn check_size_of_steelval() { assert_eq!(std::mem::size_of::(), 16); diff --git a/crates/steel-core/src/values/lists.rs b/crates/steel-core/src/values/lists.rs index ca0a0362b..f93f07e2e 100644 --- a/crates/steel-core/src/values/lists.rs +++ b/crates/steel-core/src/values/lists.rs @@ -26,6 +26,12 @@ impl Pair { } } +impl std::fmt::Debug for Pair { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({} . {})", &self.car, &self.cdr) + } +} + #[cfg(feature = "without-drop-protection")] type DropHandlerChoice = im_lists::handler::DefaultDropHandler; #[cfg(not(feature = "without-drop-protection"))] From 6714d7f6d2af833413d7c1dfee9a7acd7f81e2dc Mon Sep 17 00:00:00 2001 From: mattwparas Date: Fri, 8 Dec 2023 22:12:21 -0800 Subject: [PATCH 03/16] defmacro wip --- crates/steel-core/src/compiler/modules.rs | 92 ++++++++++++++----- .../steel-core/src/parser/expand_visitor.rs | 48 +++++++++- crates/steel-core/src/primitives/lists.rs | 3 +- crates/steel-parser/src/lexer.rs | 16 +++- crates/steel-parser/src/parser.rs | 2 +- 5 files changed, 131 insertions(+), 30 deletions(-) diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index 10d6f3634..66c2a0f93 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -9,7 +9,10 @@ use crate::{ }, parser::{ ast::{AstTools, Atom, Begin, Define, ExprKind, List, Quote}, - expand_visitor::{expand_kernel, expand_kernel_in_env}, + expand_visitor::{ + expand_kernel, expand_kernel_in_env, expand_kernel_in_env_with_allowed, + expand_kernel_in_env_with_change, + }, interner::InternedString, kernel::Kernel, parser::{ @@ -18,7 +21,7 @@ use crate::{ }, tokens::TokenType, }, - steel_vm::{engine::ModuleContainer, transducers::interleave}, + steel_vm::{builtin::BuiltInModule, engine::ModuleContainer, transducers::interleave}, }; use crate::{parser::expand_visitor::Expander, rvals::Result}; @@ -490,33 +493,66 @@ impl ModuleManager { .filter(|x| x.for_syntax) .map(|x| x.path.get_path()) { - let (module, mut in_scope_macros) = Self::find_in_scope_macros( + let (module, mut in_scope_macros, mut name_mangler) = Self::find_in_scope_macros( &self.compiled_modules, require_for_syntax.as_ref(), &mut mangled_asts, ); - // dbg!(&in_scope_macros); - - // for (key, value) in &mut in_scope_macros { - // for line in value.exprs_mut() { - // println!("{}", line); - // } - // } - - // ast = ast.into_iter().map(|x| ) - - // ast.pretty_print(); + let kernel_macros_in_scope: HashSet<_> = + module.provides_for_syntax.iter().cloned().collect(); ast = ast .into_iter() .map(|x| { + // @matt 12/8/2023 + // The easiest thing to do here, is to go to the other module, and find + // what defmacros have been exposed on the require for syntax. Once those + // have been found, we run a pass with kernel expansion, limiting the + // expander to only use the macros that we've exposed. After that, + // we run the expansion again, using the full suite of defmacro capabilities. + // + // The question that remains - how to define the neat phases of what kinds + // of macros can expand into what? Can defmacro -> syntax-rules -> defmacro? + // This could eventually prove to be cumbersome, but it is still early + // for defmacro. Plus, I need to create a syntax-case or syntax-parse + // frontend before the defmacro style macros become too pervasive. + // + // TODO: Replicate this behavior over to builtin modules + // First expand the in scope macros // These are macros let mut expander = Expander::new(&in_scope_macros); - let first_round_expanded = expander.expand(x)?; + let mut first_round_expanded = expander.expand(x)?; + let mut changed = false; + + (first_round_expanded, changed) = expand_kernel_in_env_with_allowed( + first_round_expanded, + kernel.as_mut(), + // We don't need to expand those here + ModuleContainer::default(), + module.name.to_str().unwrap().to_string(), + &kernel_macros_in_scope, + )?; - if expander.changed { + // If the kernel expander expanded into something - go ahead + // and expand all of the macros in this + if changed || expander.changed { + // Expand here? + first_round_expanded = expand(first_round_expanded, &module.macro_map)?; + + // Probably don't need this + (first_round_expanded, changed) = expand_kernel_in_env_with_change( + first_round_expanded, + kernel.as_mut(), + ModuleContainer::default(), + module.name.to_str().unwrap().to_string(), + )?; + + name_mangler.visit(&mut first_round_expanded); + } + + if expander.changed || changed { expand(first_round_expanded, &module.macro_map) } else { Ok(first_round_expanded) @@ -553,7 +589,11 @@ impl ModuleManager { compiled_modules: &'a HashMap, require_for_syntax: &'a PathBuf, mangled_asts: &'a mut Vec, - ) -> (&'a CompiledModule, HashMap) { + ) -> ( + &'a CompiledModule, + HashMap, + NameMangler, + ) { let module = compiled_modules .get(require_for_syntax) .expect(&format!("Module missing!: {:?}", require_for_syntax)); @@ -595,11 +635,11 @@ impl ModuleManager { }) .collect::>(); // Check what macros are in scope here - debug!( - "In scope macros: {:#?}", - in_scope_macros.keys().collect::>() - ); - (module, in_scope_macros) + // println!( + // "In scope macros: {:#?}", + // in_scope_macros.keys().collect::>() + // ); + (module, in_scope_macros, name_mangler) } #[cfg(not(feature = "modules"))] @@ -1492,7 +1532,7 @@ impl<'a> ModuleBuilder<'a> { .filter(|x| x.for_syntax) .map(|x| x.path.get_path()) { - let (module, in_scope_macros) = ModuleManager::find_in_scope_macros( + let (module, in_scope_macros, name_mangler) = ModuleManager::find_in_scope_macros( self.compiled_modules, require_for_syntax.as_ref(), &mut mangled_asts, @@ -1501,10 +1541,16 @@ impl<'a> ModuleBuilder<'a> { ast = ast .into_iter() .map(|x| { + for (key, _) in &in_scope_macros { + println!("1513 in scope macro Macro found: {}", key); + } // First expand the in scope macros // These are macros let mut expander = Expander::new(&in_scope_macros); let first_round_expanded = expander.expand(x)?; + for (key, _) in &module.macro_map { + println!("1520 Macro found: {}", key); + } if expander.changed { expand(first_round_expanded, &module.macro_map) diff --git a/crates/steel-core/src/parser/expand_visitor.rs b/crates/steel-core/src/parser/expand_visitor.rs index eb532c969..7e334f966 100644 --- a/crates/steel-core/src/parser/expand_visitor.rs +++ b/crates/steel-core/src/parser/expand_visitor.rs @@ -26,7 +26,7 @@ use super::{ }; use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use crate::parser::expander::SteelMacro; @@ -209,6 +209,43 @@ impl<'a> ConsumingVisitor for Expander<'a> { } } +pub fn expand_kernel_in_env_with_allowed( + expr: ExprKind, + kernel: Option<&mut Kernel>, + builtin_modules: ModuleContainer, + env: String, + allowed: &HashSet, +) -> Result<(ExprKind, bool)> { + let mut expander = KernelExpander { + map: kernel, + changed: false, + builtin_modules, + environment: Some(Cow::from(env)), + depth: 0, + allowed_macros: Some(allowed), + }; + + expander.visit(expr).map(|x| (x, expander.changed)) +} + +pub fn expand_kernel_in_env_with_change( + expr: ExprKind, + kernel: Option<&mut Kernel>, + builtin_modules: ModuleContainer, + env: String, +) -> Result<(ExprKind, bool)> { + let mut expander = KernelExpander { + map: kernel, + changed: false, + builtin_modules, + environment: Some(Cow::from(env)), + depth: 0, + allowed_macros: None, + }; + + expander.visit(expr).map(|x| (x, expander.changed)) +} + pub fn expand_kernel_in_env( expr: ExprKind, kernel: Option<&mut Kernel>, @@ -221,6 +258,7 @@ pub fn expand_kernel_in_env( builtin_modules, environment: Some(Cow::from(env)), depth: 0, + allowed_macros: None, } .visit(expr) } @@ -236,6 +274,7 @@ pub fn expand_kernel( builtin_modules, environment: None, depth: 0, + allowed_macros: None, } .visit(expr) } @@ -246,6 +285,7 @@ pub struct KernelExpander<'a> { builtin_modules: ModuleContainer, environment: Option>, depth: usize, + allowed_macros: Option<&'a HashSet>, } impl<'a> KernelExpander<'a> { @@ -256,6 +296,7 @@ impl<'a> KernelExpander<'a> { builtin_modules, environment: None, depth: 0, + allowed_macros: None, } } @@ -714,6 +755,11 @@ impl<'a> ConsumingVisitor for KernelExpander<'a> { if map .contains_syntax_object_macro(&s, self.environment.as_ref().map(|x| x.as_ref())) + && self + .allowed_macros + .as_ref() + .map(|x| x.contains(&s)) + .unwrap_or(true) { let expanded = map.expand_syntax_object( &s, diff --git a/crates/steel-core/src/primitives/lists.rs b/crates/steel-core/src/primitives/lists.rs index c63de373b..d6d2e6561 100644 --- a/crates/steel-core/src/primitives/lists.rs +++ b/crates/steel-core/src/primitives/lists.rs @@ -638,7 +638,8 @@ mod list_operation_tests { fn cons_test_normal_input() { let mut args = [SteelVal::IntV(1), SteelVal::IntV(2)]; let res = cons(&mut args); - let expected = SteelVal::ListV(vec![SteelVal::IntV(1), SteelVal::IntV(2)].into()); + + let expected = SteelVal::Pair(Gc::new(Pair::cons(SteelVal::IntV(1), SteelVal::IntV(2)))); assert_eq!(res.unwrap(), expected); } diff --git a/crates/steel-parser/src/lexer.rs b/crates/steel-parser/src/lexer.rs index 5addc468a..e70dd8d37 100644 --- a/crates/steel-parser/src/lexer.rs +++ b/crates/steel-parser/src/lexer.rs @@ -74,8 +74,6 @@ impl<'a> Lexer<'a> { let mut buf = String::new(); while let Some(&c) = self.chars.peek() { self.eat(); - // println!("{}", c); - // println!("{:?}", buf); match c { '"' => return Ok(TokenType::StringLiteral(buf)), '\\' => match self.chars.peek() { @@ -104,6 +102,11 @@ impl<'a> Lexer<'a> { buf.push('\r'); } + Some('0') => { + self.eat(); + buf.push('\0'); + } + _ => return Err(TokenError::InvalidEscape), }, _ => buf.push(c), @@ -399,8 +402,6 @@ impl<'a> Iterator for Lexer<'a> { type Item = Result>; fn next(&mut self) -> Option { - // self.consume_whitespace_and_comments_until_next_input(); - // Crunch until the next input self.consume_whitespace(); @@ -779,6 +780,13 @@ mod lexer_tests { println!("{:#?}", res); } + #[test] + fn lex_string_with_escape_chars() { + let s = TokenStream::new("\"\0\0\0\"", true, None); + let res: Vec> = s.collect(); + println!("{:#?}", res); + } + #[test] fn scheme_statement() { let s = TokenStream::new("(apples (function a b) (+ a b))", true, None); diff --git a/crates/steel-parser/src/parser.rs b/crates/steel-parser/src/parser.rs index 1699e1083..366817bf2 100644 --- a/crates/steel-parser/src/parser.rs +++ b/crates/steel-parser/src/parser.rs @@ -267,7 +267,7 @@ pub type Result = result::Result; fn tokentype_error_to_parse_error(t: &Token<'_, InternedString>) -> ParseError { if let TokenType::Error = t.ty { - // println!("Found an error: {}", t); + println!("Found an error: {}", t.typ()); if t.source.starts_with('\"') { ParseError::IncompleteString(t.source.to_string(), t.span, None) From 7a4487b1695c456471c6b86fb7baf5ffa670dde7 Mon Sep 17 00:00:00 2001 From: mattwparas Date: Sat, 9 Dec 2023 19:42:10 -0800 Subject: [PATCH 04/16] performance improvements --- crates/example-dylib/src/lib.rs | 2 +- crates/steel-core/src/compiler/compiler.rs | 34 +- crates/steel-core/src/compiler/constants.rs | 65 +++- crates/steel-core/src/compiler/modules.rs | 49 ++- .../src/compiler/passes/analysis.rs | 63 +-- .../steel-core/src/compiler/passes/mangle.rs | 2 +- crates/steel-core/src/compiler/passes/mod.rs | 4 +- .../steel-core/src/parser/expand_visitor.rs | 8 +- crates/steel-core/src/parser/expander.rs | 2 +- .../steel-core/src/parser/replace_idents.rs | 8 +- .../steel-core/src/parser/tryfrom_visitor.rs | 8 +- crates/steel-core/src/parser/visitors.rs | 8 +- crates/steel-core/src/rvals.rs | 1 + .../src/scheme/modules/parameters.scm | 10 +- crates/steel-core/src/steel_vm/builtin.rs | 360 +++++++++++++++--- .../src/steel_vm/const_evaluation.rs | 4 +- crates/steel-core/src/steel_vm/engine.rs | 62 ++- crates/steel-core/src/steel_vm/primitives.rs | 9 +- crates/steel-core/src/steel_vm/vm.rs | 4 + crates/steel-core/src/values/closed.rs | 19 +- crates/steel-parser/src/ast.rs | 37 +- crates/steel-parser/src/parser.rs | 6 +- 22 files changed, 567 insertions(+), 198 deletions(-) diff --git a/crates/example-dylib/src/lib.rs b/crates/example-dylib/src/lib.rs index 41b550818..cc5c8ade6 100644 --- a/crates/example-dylib/src/lib.rs +++ b/crates/example-dylib/src/lib.rs @@ -27,7 +27,7 @@ pub fn generate_module() -> *mut BuiltInModule { #[no_mangle] pub fn build_module(module: &mut BuiltInModule) { - module.set_name("external-dylib".to_string()); + // module.set_name("external-dylib".to_string()); module.register_value("outside-value", SteelVal::StringV("Hello world!".into())); module.register_fn("hidden-function", hidden_function); diff --git a/crates/steel-core/src/compiler/compiler.rs b/crates/steel-core/src/compiler/compiler.rs index 76d94b672..91da044c4 100644 --- a/crates/steel-core/src/compiler/compiler.rs +++ b/crates/steel-core/src/compiler/compiler.rs @@ -633,25 +633,24 @@ impl Compiler { .remove_unused_globals_with_prefix("mangler", &self.macro_env, &self.module_manager) .lift_pure_local_functions() .lift_all_local_functions(); - // .remove_unused_globals_with_prefix("manglersteel/"); // debug!("About to expand defines"); log::debug!(target: "expansion-phase", "Flattening begins, converting internal defines to let expressions"); + let mut analysis = semantic.into_analysis(); + let mut expanded_statements = flatten_begins_and_expand_defines(expanded_statements); // After define expansion, we'll want this RenameShadowedVariables::rename_shadowed_vars(&mut expanded_statements); - let mut analysis = Analysis::from_exprs(&expanded_statements); + analysis.fresh_from_exprs(&expanded_statements); analysis.populate_captures(&expanded_statements); let mut semantic = SemanticAnalysis::from_analysis(&mut expanded_statements, analysis); semantic.refresh_variables(); - semantic.flatten_anonymous_functions(); - semantic.refresh_variables(); // Replace mutation with boxes @@ -660,18 +659,20 @@ impl Compiler { semantic.replace_mutable_captured_variables_with_boxes(); - if log_enabled!(log::Level::Debug) { - debug!( - "Successfully expanded defines: {:?}", - expanded_statements - .iter() - .map(|x| x.to_string()) - .collect::>() - ); - } + // if log_enabled!(log::Level::Debug) { + // debug!( + // "Successfully expanded defines: {:?}", + // expanded_statements + // .iter() + // .map(|x| x.to_string()) + // .collect::>() + // ); + // } log::debug!(target: "expansion-phase", "Expanding multiple arity functions"); + let mut analysis = semantic.into_analysis(); + // Rename them again RenameShadowedVariables::rename_shadowed_vars(&mut expanded_statements); @@ -683,11 +684,12 @@ impl Compiler { // Begin lowering anonymous function calls to lets - let mut analysis = Analysis::from_exprs(&expanded_statements); + // let mut analysis = Analysis::from_exprs(&expanded_statements); + // let mut analysis = semantic.into_analysis(); + analysis.fresh_from_exprs(&expanded_statements); analysis.populate_captures(&expanded_statements); - let mut semantic = SemanticAnalysis::from_analysis(&mut expanded_statements, analysis); - semantic.populate_captures(); + // semantic.populate_captures(); semantic.replace_anonymous_function_calls_with_plain_lets(); diff --git a/crates/steel-core/src/compiler/constants.rs b/crates/steel-core/src/compiler/constants.rs index c0f492b84..388975363 100644 --- a/crates/steel-core/src/compiler/constants.rs +++ b/crates/steel-core/src/compiler/constants.rs @@ -6,6 +6,7 @@ use crate::parser::{ parser::{ParseError, Parser}, }; +use std::collections::HashMap; use std::{cell::RefCell, rc::Rc}; // TODO add the serializing and deserializing for constants @@ -14,7 +15,10 @@ use serde::{Deserialize, Serialize}; // Shared constant map - for repeated in memory execution of a program, this is going to share the same // underlying representation. #[derive(Debug, PartialEq)] -pub struct ConstantMap(Rc>>); +pub struct ConstantMap { + map: Rc>>, + values: Rc>>, +} #[derive(Serialize, Deserialize)] pub struct SerializableConstantMap(Vec); @@ -31,13 +35,19 @@ impl Default for ConstantMap { impl Clone for ConstantMap { fn clone(&self) -> Self { - Self(Rc::clone(&self.0)) + Self { + values: Rc::clone(&self.values), + map: Rc::clone(&self.map), + } } } impl ConstantMap { pub fn new() -> ConstantMap { - ConstantMap(Rc::new(RefCell::new(Vec::new()))) + ConstantMap { + values: Rc::new(RefCell::new(Vec::new())), + map: Rc::new(RefCell::new(HashMap::new())), + } } pub(crate) fn into_serializable_map(self) -> SerializableConstantMap { @@ -45,7 +55,7 @@ impl ConstantMap { } pub fn to_serializable_vec(&self) -> Vec { - self.0 + self.values .borrow() .iter() .cloned() @@ -56,18 +66,29 @@ impl ConstantMap { // There might be a better way of doing this - but provide this as an option // in the event we want a deep clone of the constant map - pub fn deep_clone(&self) -> ConstantMap { - ConstantMap(Rc::new(RefCell::new( - self.0.borrow().iter().cloned().collect(), - ))) - } + // pub fn deep_clone(&self) -> ConstantMap { + // ConstantMap(Rc::new(RefCell::new( + // self.0.borrow().iter().cloned().collect(), + // ))) + // } pub fn from_vec(vec: Vec) -> ConstantMap { - ConstantMap(Rc::new(RefCell::new(vec))) + // ConstantMap(Rc::new(RefCell::new(vec))) + + ConstantMap { + map: Rc::new(RefCell::new( + vec.clone() + .into_iter() + .enumerate() + .map(|x| (x.1, x.0)) + .collect(), + )), + values: Rc::new(RefCell::new(vec)), + } } fn to_constant_expr_map(&self) -> Vec { - self.0 + self.values .borrow() .iter() .map(|x| match x { @@ -113,7 +134,7 @@ impl ConstantMap { // Ok(SteelVal::try_from(parsed[0].clone()).unwrap()) }) .collect::>>() - .map(|x| ConstantMap(Rc::new(RefCell::new(x)))) + .map(Self::from_vec) } // pub fn from_bytes(encoded: &[u8]) -> ConstantMap { @@ -124,18 +145,22 @@ impl ConstantMap { impl ConstantMap { pub fn add(&mut self, val: SteelVal) -> usize { let idx = self.len(); - self.0.borrow_mut().push(val); + self.values.borrow_mut().push(val.clone()); + + // TODO: Consider just storing the hash code, not the actual value. + self.map.borrow_mut().insert(val, idx); + idx } // Fallible #[inline(always)] pub fn get(&self, idx: usize) -> SteelVal { - self.0.borrow()[idx].clone() + self.values.borrow()[idx].clone() } pub fn try_get(&self, idx: usize) -> Option { - self.0.borrow().get(idx).cloned() + self.values.borrow().get(idx).cloned() } // Replace with existing constants if they already exist @@ -163,7 +188,7 @@ impl ConstantMap { }; } - let idx = { self.0.borrow_mut().iter().position(|x| x == &val) }; + let idx = self.map.borrow_mut().get(&val).copied(); if let Some(idx) = idx { idx @@ -173,20 +198,20 @@ impl ConstantMap { } pub fn len(&self) -> usize { - self.0.borrow().len() + self.values.borrow().len() } pub fn is_empty(&self) -> bool { - self.0.borrow().is_empty() + self.values.borrow().is_empty() } pub fn roll_back(&mut self, idx: usize) { - self.0.borrow_mut().truncate(idx); + self.values.borrow_mut().truncate(idx); } #[cfg(test)] pub fn clear(&mut self) { - self.0.borrow_mut().clear() + self.values.borrow_mut().clear() } } diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index 66c2a0f93..4951afdcd 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -549,7 +549,12 @@ impl ModuleManager { module.name.to_str().unwrap().to_string(), )?; - name_mangler.visit(&mut first_round_expanded); + // This is pretty suspect, and needs to be revisited - only the output of the + // macro expansion and not the whole thing needs to be mangled most likely. + // Otherwise, we'll run into weird stuff? + if changed { + name_mangler.visit(&mut first_round_expanded); + } } if expander.changed || changed { @@ -557,8 +562,6 @@ impl ModuleManager { } else { Ok(first_round_expanded) } - - // expand(x, &module.macro_map) }) .collect::>()?; @@ -1532,27 +1535,51 @@ impl<'a> ModuleBuilder<'a> { .filter(|x| x.for_syntax) .map(|x| x.path.get_path()) { - let (module, in_scope_macros, name_mangler) = ModuleManager::find_in_scope_macros( + let (module, in_scope_macros, mut name_mangler) = ModuleManager::find_in_scope_macros( self.compiled_modules, require_for_syntax.as_ref(), &mut mangled_asts, ); + let kernel_macros_in_scope: HashSet<_> = + module.provides_for_syntax.iter().cloned().collect(); + ast = ast .into_iter() .map(|x| { - for (key, _) in &in_scope_macros { - println!("1513 in scope macro Macro found: {}", key); - } // First expand the in scope macros // These are macros let mut expander = Expander::new(&in_scope_macros); - let first_round_expanded = expander.expand(x)?; - for (key, _) in &module.macro_map { - println!("1520 Macro found: {}", key); + let mut first_round_expanded = expander.expand(x)?; + let mut changed = false; + + // (first_round_expanded, changed) = expand_kernel_in_env_with_allowed( + // first_round_expanded, + // self.kernel.as_mut(), + // // We don't need to expand those here + // ModuleContainer::default(), + // module.name.to_str().unwrap().to_string(), + // &kernel_macros_in_scope, + // )?; + + // If the kernel expander expanded into something - go ahead + // and expand all of the macros in this + if changed || expander.changed { + // Expand here? + first_round_expanded = expand(first_round_expanded, &module.macro_map)?; + + // Probably don't need this + // (first_round_expanded, changed) = expand_kernel_in_env_with_change( + // first_round_expanded, + // self.kernel.as_mut(), + // ModuleContainer::default(), + // module.name.to_str().unwrap().to_string(), + // )?; + + // name_mangler.visit(&mut first_round_expanded); } - if expander.changed { + if expander.changed || changed { expand(first_round_expanded, &module.macro_map) } else { Ok(first_round_expanded) diff --git a/crates/steel-core/src/compiler/passes/analysis.rs b/crates/steel-core/src/compiler/passes/analysis.rs index 66c709882..7d8b7b6ac 100644 --- a/crates/steel-core/src/compiler/passes/analysis.rs +++ b/crates/steel-core/src/compiler/passes/analysis.rs @@ -25,7 +25,7 @@ use crate::{ use super::{VisitorMutControlFlow, VisitorMutRefUnit, VisitorMutUnitRef}; -use fxhash::{FxHashMap, FxHasher}; +use fxhash::{FxHashMap, FxHashSet, FxHasher}; #[derive(Clone, Copy, Debug, PartialEq)] pub enum IdentifierStatus { @@ -250,6 +250,19 @@ pub struct Analysis { } impl Analysis { + // Reuse the analysis allocation through the process! + pub fn clear(&mut self) { + self.info.clear(); + self.function_info.clear(); + self.call_info.clear(); + self.let_info.clear(); + } + + pub fn fresh_from_exprs(&mut self, exprs: &[ExprKind]) { + self.clear(); + self.run(exprs); + } + pub fn from_exprs(exprs: &[ExprKind]) -> Self { let mut analysis = Analysis::default(); analysis.run(exprs); @@ -265,7 +278,7 @@ impl Analysis { .chain(self.let_info.values().flat_map(|x| x.arguments.values())) .filter(|x| x.captured && x.mutated) .map(|x| (x.id, x.clone())) - .collect::>(); + .collect::>(); self.function_info .values_mut() @@ -2918,7 +2931,7 @@ impl<'a> VisitorMutRefUnit for FlattenAnonymousFunctionCalls<'a> { struct FunctionCallCollector<'a> { analysis: &'a Analysis, - functions: HashMap>, + functions: FxHashMap>, black_box: InternedString, context: Option, constants: ImmutableHashMap, @@ -2931,12 +2944,12 @@ impl<'a> FunctionCallCollector<'a> { exprs: &mut Vec, constants: ImmutableHashMap, should_mangle: bool, - ) -> HashMap> { + ) -> FxHashMap> { let black_box: InternedString = "#%black-box".into(); let mut collector = Self { analysis, - functions: HashMap::new(), + functions: FxHashMap::default(), context: None, black_box, constants, @@ -3078,6 +3091,10 @@ pub struct SemanticAnalysis<'a> { } impl<'a> SemanticAnalysis<'a> { + pub fn into_analysis(self) -> Analysis { + self.analysis + } + pub fn from_analysis(exprs: &'a mut Vec, analysis: Analysis) -> Self { Self { exprs, analysis } } @@ -3183,7 +3200,7 @@ impl<'a> SemanticAnalysis<'a> { *self.exprs = lifter.lifted_functions; log::debug!("Re-running the analysis after lifting local functions"); - self.analysis = Analysis::from_exprs(self.exprs); + self.analysis.fresh_from_exprs(&self.exprs); self.analysis.populate_captures(self.exprs); } @@ -3196,10 +3213,6 @@ impl<'a> SemanticAnalysis<'a> { module_manager: &mut ModuleManager, table: &mut HashSet, ) -> &mut Self { - // for identifier in table.iter() { - // println!("Table => {}", identifier); - // } - let mut replacer = ReplaceBuiltinUsagesWithReservedPrimitiveReferences::new(&self.analysis, table); @@ -3207,10 +3220,6 @@ impl<'a> SemanticAnalysis<'a> { replacer.visit(expr); } - // for identifier in replacer.identifiers_to_replace.iter() { - // println!("Replaced => {}", identifier); - // } - let mut macro_replacer = ReplaceBuiltinUsagesInsideMacros { identifiers_to_replace: replacer.identifiers_to_replace, analysis: &self.analysis, @@ -3242,9 +3251,7 @@ impl<'a> SemanticAnalysis<'a> { macro_replacer.visit(expr); } - self.analysis = Analysis::from_exprs(self.exprs); - - // replace.vi + self.analysis.fresh_from_exprs(self.exprs); self } @@ -3393,7 +3400,7 @@ impl<'a> SemanticAnalysis<'a> { log::debug!("Re-running the semantic analysis after removing unused globals"); - self.analysis = Analysis::from_exprs(self.exprs); + self.analysis.fresh_from_exprs(self.exprs); self } @@ -3437,7 +3444,7 @@ impl<'a> SemanticAnalysis<'a> { if re_run_analysis { log::debug!("Re-running the semantic analysis after modifying let call sites"); - self.analysis = Analysis::from_exprs(self.exprs); + self.analysis.fresh_from_exprs(self.exprs); } self @@ -3447,7 +3454,7 @@ impl<'a> SemanticAnalysis<'a> { &mut self, constants: ImmutableHashMap, should_mangle: bool, - ) -> HashMap> { + ) -> FxHashMap> { let map = FunctionCallCollector::mangle( &self.analysis, &mut self.exprs, @@ -3460,7 +3467,7 @@ impl<'a> SemanticAnalysis<'a> { .iter() .filter(|(_, v)| v.is_empty()) .map(|x| x.0.clone()) - .collect::>(); + .collect::>(); // Only constant evaluatable functions should be ones that references _other_ const functions map.into_iter() @@ -3525,7 +3532,8 @@ impl<'a> SemanticAnalysis<'a> { if re_run_analysis { log::debug!("Re-running the semantic analysis after modifications"); - self.analysis = Analysis::from_exprs(self.exprs); + // self.analysis = Analysis::from_exprs(self.exprs); + self.analysis.fresh_from_exprs(self.exprs); } self @@ -3581,7 +3589,7 @@ impl<'a> SemanticAnalysis<'a> { if re_run_analysis { log::debug!("Re-running the semantic analysis after modifications"); - self.analysis = Analysis::from_exprs(self.exprs); + self.analysis.fresh_from_exprs(self.exprs); } self @@ -3592,7 +3600,7 @@ impl<'a> SemanticAnalysis<'a> { RefreshVars.visit(expr); } - self.analysis = Analysis::from_exprs(self.exprs); + self.analysis.fresh_from_exprs(self.exprs); self } @@ -3706,7 +3714,7 @@ impl<'a> SemanticAnalysis<'a> { "Re-running the semantic analysis after modifications during lambda lifting" ); - self.analysis = Analysis::from_exprs(self.exprs); + self.analysis.fresh_from_exprs(self.exprs); self.analysis.populate_captures(self.exprs); } @@ -3826,8 +3834,9 @@ impl<'a> SemanticAnalysis<'a> { "Re-running the semantic analysis after modifications during lambda lifting" ); - self.analysis = Analysis::from_exprs(self.exprs); - self.analysis.populate_captures(self.exprs); + self.analysis.fresh_from_exprs(self.exprs); + // = Analysis::from_exprs(self.exprs); + // self.analysis.populate_captures(self.exprs); } self diff --git a/crates/steel-core/src/compiler/passes/mangle.rs b/crates/steel-core/src/compiler/passes/mangle.rs index 212b5a822..58e39d0ac 100644 --- a/crates/steel-core/src/compiler/passes/mangle.rs +++ b/crates/steel-core/src/compiler/passes/mangle.rs @@ -79,7 +79,7 @@ impl<'a> VisitorMutRefUnit for NameUnMangler<'a> { } pub struct NameMangler { - globals: HashSet, + pub(crate) globals: HashSet, prefix: String, } diff --git a/crates/steel-core/src/compiler/passes/mod.rs b/crates/steel-core/src/compiler/passes/mod.rs index 60053977e..fedff42c6 100644 --- a/crates/steel-core/src/compiler/passes/mod.rs +++ b/crates/steel-core/src/compiler/passes/mod.rs @@ -96,7 +96,7 @@ pub trait Folder { } #[inline] - fn visit_macro(&mut self, m: Macro) -> ExprKind { + fn visit_macro(&mut self, m: Box) -> ExprKind { ExprKind::Macro(m) } @@ -112,7 +112,7 @@ pub trait Folder { } #[inline] - fn visit_syntax_rules(&mut self, l: SyntaxRules) -> ExprKind { + fn visit_syntax_rules(&mut self, l: Box) -> ExprKind { ExprKind::SyntaxRules(l) } diff --git a/crates/steel-core/src/parser/expand_visitor.rs b/crates/steel-core/src/parser/expand_visitor.rs index 7e334f966..60ac66f3c 100644 --- a/crates/steel-core/src/parser/expand_visitor.rs +++ b/crates/steel-core/src/parser/expand_visitor.rs @@ -135,7 +135,7 @@ impl<'a> ConsumingVisitor for Expander<'a> { Ok(ExprKind::Quote(quote)) } - fn visit_macro(&mut self, m: super::ast::Macro) -> Self::Output { + fn visit_macro(&mut self, m: Box) -> Self::Output { stop!(BadSyntax => format!("unexpected macro definition in expand visitor: {}", m); m.location.span) } @@ -178,7 +178,7 @@ impl<'a> ConsumingVisitor for Expander<'a> { Ok(ExprKind::List(l)) } - fn visit_syntax_rules(&mut self, l: super::ast::SyntaxRules) -> Self::Output { + fn visit_syntax_rules(&mut self, l: Box) -> Self::Output { dbg!(l.to_string()); stop!(Generic => "unexpected syntax-rules definition"; l.location.span) @@ -623,7 +623,7 @@ impl<'a> ConsumingVisitor for KernelExpander<'a> { Ok(ExprKind::Quote(quote)) } - fn visit_macro(&mut self, m: super::ast::Macro) -> Self::Output { + fn visit_macro(&mut self, m: Box) -> Self::Output { stop!(BadSyntax => format!("unexpected macro definition in kernel expander: {}", m); m.location.span) } @@ -947,7 +947,7 @@ impl<'a> ConsumingVisitor for KernelExpander<'a> { Ok(ExprKind::List(l)) } - fn visit_syntax_rules(&mut self, l: super::ast::SyntaxRules) -> Self::Output { + fn visit_syntax_rules(&mut self, l: Box) -> Self::Output { dbg!(l.to_string()); stop!(Generic => "unexpected syntax-rules definition"; l.location.span) diff --git a/crates/steel-core/src/parser/expander.rs b/crates/steel-core/src/parser/expander.rs index ad7557d61..a1c6640ca 100644 --- a/crates/steel-core/src/parser/expander.rs +++ b/crates/steel-core/src/parser/expander.rs @@ -178,7 +178,7 @@ impl SteelMacro { self.mangled } - pub fn parse_from_ast_macro(ast_macro: Macro) -> Result { + pub fn parse_from_ast_macro(ast_macro: Box) -> Result { let name = *ast_macro .name .atom_identifier_or_else(throw!(BadSyntax => "macros only currently support diff --git a/crates/steel-core/src/parser/replace_idents.rs b/crates/steel-core/src/parser/replace_idents.rs index b3d0e330a..a779914b9 100644 --- a/crates/steel-core/src/parser/replace_idents.rs +++ b/crates/steel-core/src/parser/replace_idents.rs @@ -466,7 +466,7 @@ impl<'a> ConsumingVisitor for ReplaceExpressions<'a> { Ok(ExprKind::Quote(quote)) } - fn visit_macro(&mut self, m: super::ast::Macro) -> Self::Output { + fn visit_macro(&mut self, m: Box) -> Self::Output { stop!(BadSyntax => format!("unexpected macro definition: {}", m); m.location.span) } @@ -497,7 +497,7 @@ impl<'a> ConsumingVisitor for ReplaceExpressions<'a> { Ok(ExprKind::List(l)) } - fn visit_syntax_rules(&mut self, l: super::ast::SyntaxRules) -> Self::Output { + fn visit_syntax_rules(&mut self, l: Box) -> Self::Output { dbg!(l.to_string()); stop!(Generic => "unexpected syntax-rules definition"; l.location.span) @@ -604,7 +604,7 @@ impl ConsumingVisitor for RewriteSpan { Ok(ExprKind::Quote(quote)) } - fn visit_macro(&mut self, m: super::ast::Macro) -> Self::Output { + fn visit_macro(&mut self, m: Box) -> Self::Output { stop!(BadSyntax => format!("unexpected macro definition: {}", m); m.location.span) } @@ -630,7 +630,7 @@ impl ConsumingVisitor for RewriteSpan { Ok(ExprKind::List(l)) } - fn visit_syntax_rules(&mut self, l: super::ast::SyntaxRules) -> Self::Output { + fn visit_syntax_rules(&mut self, l: Box) -> Self::Output { dbg!(l.to_string()); stop!(Generic => "unexpected syntax-rules definition"; l.location.span) diff --git a/crates/steel-core/src/parser/tryfrom_visitor.rs b/crates/steel-core/src/parser/tryfrom_visitor.rs index fa57a62e0..26d704658 100644 --- a/crates/steel-core/src/parser/tryfrom_visitor.rs +++ b/crates/steel-core/src/parser/tryfrom_visitor.rs @@ -101,7 +101,7 @@ impl ConsumingVisitor for TryFromExprKindForSteelVal { } } - fn visit_macro(&mut self, m: super::ast::Macro) -> Self::Output { + fn visit_macro(&mut self, m: Box) -> Self::Output { // TODO Ok(SteelVal::ListV( @@ -127,7 +127,7 @@ impl ConsumingVisitor for TryFromExprKindForSteelVal { Ok(items?.into()) } - fn visit_syntax_rules(&mut self, s: super::ast::SyntaxRules) -> Self::Output { + fn visit_syntax_rules(&mut self, s: Box) -> Self::Output { Ok(SteelVal::ListV( vec![ SteelVal::SymbolV("syntax-rules".into()), @@ -318,7 +318,7 @@ impl ConsumingVisitor for SyntaxObjectFromExprKind { // } } - fn visit_macro(&mut self, _m: super::ast::Macro) -> Self::Output { + fn visit_macro(&mut self, _m: Box) -> Self::Output { // TODO stop!(Generic => "internal compiler error - could not translate macro to steel value") } @@ -358,7 +358,7 @@ impl ConsumingVisitor for SyntaxObjectFromExprKind { Ok(Syntax::proto(raw, items.into(), span).into()) } - fn visit_syntax_rules(&mut self, _s: super::ast::SyntaxRules) -> Self::Output { + fn visit_syntax_rules(&mut self, _s: Box) -> Self::Output { // Ok(SteelVal::ListV( // vec![ // SteelVal::SymbolV("syntax-rules".into()), diff --git a/crates/steel-core/src/parser/visitors.rs b/crates/steel-core/src/parser/visitors.rs index 365d89235..305f4f5df 100644 --- a/crates/steel-core/src/parser/visitors.rs +++ b/crates/steel-core/src/parser/visitors.rs @@ -141,10 +141,10 @@ pub trait ConsumingVisitor { fn visit_begin(&mut self, begin: Begin) -> Self::Output; fn visit_return(&mut self, r: Box) -> Self::Output; fn visit_quote(&mut self, quote: Box) -> Self::Output; - fn visit_macro(&mut self, m: Macro) -> Self::Output; + fn visit_macro(&mut self, m: Box) -> Self::Output; fn visit_atom(&mut self, a: Atom) -> Self::Output; fn visit_list(&mut self, l: List) -> Self::Output; - fn visit_syntax_rules(&mut self, l: SyntaxRules) -> Self::Output; + fn visit_syntax_rules(&mut self, l: Box) -> Self::Output; fn visit_set(&mut self, s: Box) -> Self::Output; fn visit_require(&mut self, s: Require) -> Self::Output; fn visit_let(&mut self, l: Box) -> Self::Output; @@ -176,10 +176,10 @@ pub trait ConsumingVisitorRef { fn visit_begin(&self, begin: Begin) -> Self::Output; fn visit_return(&self, r: Box) -> Self::Output; fn visit_quote(&self, quote: Box) -> Self::Output; - fn visit_macro(&self, m: Macro) -> Self::Output; + fn visit_macro(&self, m: Box) -> Self::Output; fn visit_atom(&self, a: Atom) -> Self::Output; fn visit_list(&self, l: List) -> Self::Output; - fn visit_syntax_rules(&self, l: SyntaxRules) -> Self::Output; + fn visit_syntax_rules(&self, l: Box) -> Self::Output; fn visit_set(&self, s: Box) -> Self::Output; fn visit_require(&self, s: Require) -> Self::Output; fn visit_let(&self, l: Box) -> Self::Output; diff --git a/crates/steel-core/src/rvals.rs b/crates/steel-core/src/rvals.rs index 50812e503..ca5ae79e4 100644 --- a/crates/steel-core/src/rvals.rs +++ b/crates/steel-core/src/rvals.rs @@ -1437,6 +1437,7 @@ impl Hash for SteelVal { CharV(c) => c.hash(state), ListV(l) => l.hash(state), CustomStruct(s) => s.hash(state), + BigNum(n) => n.hash(state), // Pair(cell) => { // cell.hash(state); // } diff --git a/crates/steel-core/src/scheme/modules/parameters.scm b/crates/steel-core/src/scheme/modules/parameters.scm index fcd9950ba..5e62371a5 100644 --- a/crates/steel-core/src/scheme/modules/parameters.scm +++ b/crates/steel-core/src/scheme/modules/parameters.scm @@ -91,9 +91,9 @@ (define winders '()) -(define list-tail drop) +; (define list-tail drop) -(struct Pair (left right)) +; (struct Pair (left right)) (define common-tail (lambda (x y) @@ -112,14 +112,14 @@ ;; TODO: This is probably wrong! ; (displayln "FIRST" ls) (set! winders (cdr ls)) - ((Pair-right (car ls))) + ((cdr (car ls))) (f (cdr ls))))) (let f ([ls new]) (when (not (equal? ls tail)) (begin ; (displayln "SECOND" ls) (f (cdr ls)) - ((Pair-left (car ls))) + ((car (car ls))) (set! winders ls))))))) (struct Continuation (func) @@ -145,7 +145,7 @@ (define dynamic-wind (lambda (in body out) (in) - (set! winders (cons (Pair in out) winders)) + (set! winders (cons (cons in out) winders)) (let ([ans* (call-with-exception-handler (lambda (err) ;; Catch the exception on the way out diff --git a/crates/steel-core/src/steel_vm/builtin.rs b/crates/steel-core/src/steel_vm/builtin.rs index 4876f9311..639d155a4 100644 --- a/crates/steel-core/src/steel_vm/builtin.rs +++ b/crates/steel-core/src/steel_vm/builtin.rs @@ -31,14 +31,21 @@ use once_cell::sync::Lazy; /// TODO: @Matt - We run the risk of running into memory leaks here when exposing external mutable /// structs. This should be more properly documented. #[derive(Clone, Debug)] -#[repr(C)] pub struct BuiltInModule { + module: Rc>, +} + +#[derive(Clone, Debug)] +struct BuiltInModuleRepr { pub(crate) name: Rc, values: HashMap, SteelVal>, docs: Box, version: &'static str, // Add the metadata separate from the pointer, keeps the pointer slim fn_ptr_table: HashMap<*const FunctionSignature, FunctionSignatureMetadata>, + // We don't need to generate this every time, just need to + // clone it? + generated_expression: RefCell>, } #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -91,7 +98,7 @@ impl Custom for BuiltInModule {} impl RegisterValue for BuiltInModule { fn register_value_inner(&mut self, name: &str, value: SteelVal) -> &mut Self { - self.values.insert(name.into(), value); + self.module.borrow_mut().values.insert(name.into(), value); self } } @@ -121,7 +128,7 @@ pub struct NativeFunctionDefinition { pub is_const: bool, } -impl BuiltInModule { +impl BuiltInModuleRepr { pub fn new>>(name: T) -> Self { Self { name: name.into(), @@ -129,50 +136,40 @@ impl BuiltInModule { docs: Box::new(InternalDocumentation::new()), version: env!("CARGO_PKG_VERSION"), fn_ptr_table: HashMap::new(), + generated_expression: RefCell::new(None), } } - pub fn documentation(&self) -> &InternalDocumentation { - &self.docs - } - - pub fn set_name(&mut self, name: String) { - self.name = name.into(); - } - - pub fn register_native_fn( - &mut self, - name: &'static str, - func: fn(&[SteelVal]) -> Result, - arity: Arity, - ) -> &mut Self { - // Just automatically add it to the function pointer table to help out with searching - self.add_to_fn_ptr_table(func, FunctionSignatureMetadata::new(name, arity, false)); - self.register_value(name, SteelVal::FuncV(func)) - } - - pub fn register_native_fn_definition( - &mut self, - definition: NativeFunctionDefinition, - ) -> &mut Self { - self.add_to_fn_ptr_table( - definition.func, - FunctionSignatureMetadata::new(definition.name, definition.arity, definition.is_const), - ); - - if let Some(doc) = definition.doc { - self.register_doc(definition.name, doc); - } - - self.register_value(definition.name, SteelVal::FuncV(definition.func)); - - self - } - - pub fn check_compatibility(self: &BuiltInModule) -> bool { - // self.version == env!("CARGO_PKG_VERSION") - true - } + // pub fn register_native_fn( + // &mut self, + // name: &'static str, + // func: fn(&[SteelVal]) -> Result, + // arity: Arity, + // ) -> &mut Self { + // // Just automatically add it to the function pointer table to help out with searching + // self.add_to_fn_ptr_table(func, FunctionSignatureMetadata::new(name, arity, false)); + // self.register_value(name, SteelVal::FuncV(func)) + // } + + // pub fn register_native_fn_definition( + // &mut self, + // definition: NativeFunctionDefinition, + // ) -> &mut Self { + // self.add_to_fn_ptr_table( + // definition.func, + // FunctionSignatureMetadata::new(definition.name, definition.arity, definition.is_const), + // ); + // if let Some(doc) = definition.doc { + // self.register_doc(definition.name, doc); + // } + // self.register_value(definition.name, SteelVal::FuncV(definition.func)); + // self + // } + + // pub fn check_compatibility(self: &BuiltInModule) -> bool { + // // self.version == env!("CARGO_PKG_VERSION") + // true + // } pub fn contains(&self, ident: &str) -> bool { self.values.contains_key(ident) @@ -219,13 +216,12 @@ impl BuiltInModule { .collect() } - pub fn with_module(mut self, module: BuiltInModule) -> Self { - // self.values = self.values.union(module.values); - - self.values.extend(module.values.into_iter()); + pub fn with_module(&mut self, module: BuiltInModule) { + self.values = std::mem::take(&mut self.values).union(module.module.borrow().values.clone()); - self.docs.definitions.extend(module.docs.definitions); - self + self.docs + .definitions + .extend(module.module.borrow().docs.definitions.clone()); } pub fn register_type( @@ -276,7 +272,8 @@ impl BuiltInModule { /// Add a value to the module namespace. This value can be any legal SteelVal, or if you're explicitly attempting /// to compile an program for later use and don't currently have access to the functions in memory, use `SteelVal::Void` pub fn register_value(&mut self, name: &str, value: SteelVal) -> &mut Self { - self.register_value_inner(name, value) + self.values.insert(name.into(), value); + self } pub fn register_value_with_doc( @@ -367,6 +364,17 @@ impl BuiltInModule { /// Scripts can choose to include these modules directly, or opt to not, and are not as risk of clobbering their /// global namespace. pub fn to_syntax(&self, prefix: Option<&str>) -> ExprKind { + let now = std::time::Instant::now(); + + // log::debug!(target: "engine-creation", "{:p}, Creating module: {} - Prefix: {:?} - cached: {}", self, self.name, prefix, self.generated_expression.borrow().is_some()); + + // No need to generate this module multiple times - + if prefix.is_none() && self.generated_expression.borrow().is_some() { + // log::debug!(target: "engine-creation", "Getting the module from the cache!"); + + return self.generated_expression.borrow().as_ref().unwrap().clone(); + } + let module_name = self.unreadable_name(); let mut defines = self @@ -406,11 +414,261 @@ impl BuiltInModule { ))), ]))); + let res = ExprKind::Begin(crate::parser::ast::Begin::new( + defines, + SyntaxObject::default(TokenType::Begin), + )); + + // Cache the generated expression + if prefix.is_none() && self.generated_expression.borrow().is_none() { + *self.generated_expression.borrow_mut() = Some(res.clone()); + } + + // log::debug!(target: "engine-creation", "Generating expression for: {} took: {:?}", self.name, now.elapsed()); + + res + } +} + +impl BuiltInModule { + pub fn new>>(name: T) -> Self { + Self { + module: Rc::new(RefCell::new(BuiltInModuleRepr::new(name))), + } + } + + pub fn name(&self) -> Rc { + Rc::clone(&self.module.borrow().name) + } + + pub fn documentation(&self) -> std::cell::Ref<'_, InternalDocumentation> { + std::cell::Ref::map(self.module.borrow(), |x| x.docs.as_ref()) + } + + // pub fn set_name(&mut self, name: String) { + // self.name = name.into(); + // } + + pub fn register_native_fn( + &mut self, + name: &'static str, + func: fn(&[SteelVal]) -> Result, + arity: Arity, + ) -> &mut Self { + // Just automatically add it to the function pointer table to help out with searching + self.add_to_fn_ptr_table(func, FunctionSignatureMetadata::new(name, arity, false)); + self.register_value(name, SteelVal::FuncV(func)) + } + + pub fn register_native_fn_definition( + &mut self, + definition: NativeFunctionDefinition, + ) -> &mut Self { + self.add_to_fn_ptr_table( + definition.func, + FunctionSignatureMetadata::new(definition.name, definition.arity, definition.is_const), + ); + + if let Some(doc) = definition.doc { + self.register_doc(definition.name, doc); + } + + self.register_value(definition.name, SteelVal::FuncV(definition.func)); + + self + } + + pub fn check_compatibility(self: &BuiltInModule) -> bool { + // self.version == env!("CARGO_PKG_VERSION") + true + } + + pub fn contains(&self, ident: &str) -> bool { + // self.values.contains_key(ident) + self.module.borrow().contains(ident) + } + + pub(crate) fn add_to_fn_ptr_table( + &mut self, + value: FunctionSignature, + data: FunctionSignatureMetadata, + ) -> &mut Self { + // // Store this in a globally accessible place for printing + // FUNCTION_TABLE.with(|table| { + // table + // .borrow_mut() + // .insert(value as *const FunctionSignature, data) + // }); + + // // Probably don't need to store it in both places? + // self.fn_ptr_table + // .insert(value as *const FunctionSignature, data); + + // self + + self.module.borrow_mut().add_to_fn_ptr_table(value, data); + + self + } + + pub fn search(&self, value: SteelVal) -> Option { + self.module.borrow().search(value) + } + + pub fn bound_identifiers(&self) -> crate::values::lists::List { + // self.values + // .keys() + // .map(|x| SteelVal::StringV(x.to_string().into())) + // .collect() + + self.module.borrow().bound_identifiers() + } + + pub fn with_module(mut self, module: BuiltInModule) -> Self { + // self.values.extend(module.values.into_iter()); + + // self.docs.definitions.extend(module.docs.definitions); + // self + + self.module.borrow_mut().with_module(module); + self + } + + pub fn register_type( + &mut self, + predicate_name: &'static str, + ) -> &mut Self { + self.module.borrow_mut().register_type::(predicate_name); + self + } + + pub fn register_doc( + &mut self, + definition: impl Into>, + description: impl Into>, + ) -> &mut Self { + // self.docs.register_doc(definition, description.into()); + // self + + self.module + .borrow_mut() + .register_doc(definition, description); + self + } + + // pub fn docs(&self) -> + + pub fn get_doc(&self, definition: String) { + // if let Some(value) = self.docs.get(&definition) { + // println!("{value}") + // } + + self.module.borrow().get_doc(definition); + } + + pub(crate) fn unreadable_name(&self) -> String { + "%-builtin-module-".to_string() + &self.module.borrow().name + } + + /// Add a value to the module namespace. This value can be any legal SteelVal, or if you're explicitly attempting + /// to compile an program for later use and don't currently have access to the functions in memory, use `SteelVal::Void` + pub fn register_value(&mut self, name: &str, value: SteelVal) -> &mut Self { + self.register_value_inner(name, value) + } + + pub fn register_value_with_doc( + &mut self, + name: &'static str, + value: SteelVal, + doc: DocTemplate<'static>, + ) -> &mut Self { + // self.values.insert(name.into(), value); + // self.register_doc(Cow::from(name), doc); + // self + + self.module + .borrow_mut() + .register_value_with_doc(name, value, doc); + self + } + + // This _will_ panic given an incorrect value. This will be tied together by macros only allowing legal entries + pub fn get(&self, name: String) -> SteelVal { + // self.values.get(name.as_str()).unwrap().clone() + + self.module.borrow().get(name) + } + + // When we're loading dylib, we won't know anything about it until _after_ it is loaded. We don't explicitly + // want to load it before we need it, since the compiler should be able to analyze a dylib without having to + // have the dylib built and loaded into memory to do so. + pub fn dylib_to_syntax<'a>( + dylib_name: &'a str, + names: impl Iterator, + prefix: Option<&str>, + ) -> ExprKind { + let mut defines = names + .map(|x| { + // TODO: Consider a custom delimeter as well + // If we have a prefix, put the prefix at the front and append x + // Otherwise, just default to using the provided name + let name = prefix + .map(|pre| pre.to_string() + x) + .unwrap_or_else(|| x.to_string()); + + ExprKind::Define(Box::new(crate::parser::ast::Define::new( + // TODO: Add the custom prefix here + // Handling a more complex case of qualifying imports + ExprKind::atom(name), + ExprKind::List(crate::parser::ast::List::new(vec![ + ExprKind::atom(*MODULE_GET), + ExprKind::List(crate::parser::ast::List::new(vec![ + ExprKind::atom(*GET_DYLIB), + ExprKind::string_lit(dylib_name.to_string()), + ])), + ExprKind::Quote(Box::new(crate::parser::ast::Quote::new( + ExprKind::atom(x.to_string()), + SyntaxObject::default(TokenType::Quote), + ))), + ])), + SyntaxObject::default(TokenType::Define), + ))) + }) + .collect::>(); + + defines.push(ExprKind::List(crate::parser::ast::List::new(vec![ + ExprKind::atom(*MODULE_GET), + ExprKind::atom("%-builtin-module-".to_string() + "steel/constants"), + ExprKind::Quote(Box::new(crate::parser::ast::Quote::new( + ExprKind::atom(*VOID), + SyntaxObject::default(TokenType::Quote), + ))), + ]))); + ExprKind::Begin(crate::parser::ast::Begin::new( defines, SyntaxObject::default(TokenType::Begin), )) } + + /// This does the boot strapping for bundling modules + /// Rather than expose a native hash-get, the built in module above should expose a raw + /// function to fetch a dependency. It will be a packaged # with only a function to + /// fetch a function given its registered name. For instance: + /// + /// (##module-get## ##unreadable-module-name-core-lists## 'list) + /// + /// This puts the onus on the expansion of a primitive on the compiler, but now the language + /// is bootstrapped via syntax-rules and the kernel macro expansion, and other dependencies + /// are included via the usual macro expansion. + /// + /// In this way its always possible to refresh the native functions (and they don't disappear), + /// and bundles of functions from third parties don't get included immediately into the global namespace. + /// Scripts can choose to include these modules directly, or opt to not, and are not as risk of clobbering their + /// global namespace. + pub fn to_syntax(&self, prefix: Option<&str>) -> ExprKind { + self.module.borrow().to_syntax(prefix) + } } /// Documentation representation diff --git a/crates/steel-core/src/steel_vm/const_evaluation.rs b/crates/steel-core/src/steel_vm/const_evaluation.rs index a9546d9f0..3ea226cca 100644 --- a/crates/steel-core/src/steel_vm/const_evaluation.rs +++ b/crates/steel-core/src/steel_vm/const_evaluation.rs @@ -508,7 +508,7 @@ impl<'a> ConsumingVisitor for ConstantEvaluator<'a> { Ok(ExprKind::Quote(quote)) } - fn visit_macro(&mut self, _m: crate::parser::ast::Macro) -> Self::Output { + fn visit_macro(&mut self, _m: Box) -> Self::Output { stop!(Generic => "unexpected macro found in const evaluator"); } @@ -811,7 +811,7 @@ impl<'a> ConsumingVisitor for ConstantEvaluator<'a> { } } - fn visit_syntax_rules(&mut self, _l: crate::parser::ast::SyntaxRules) -> Self::Output { + fn visit_syntax_rules(&mut self, _l: Box) -> Self::Output { stop!(Generic => "unexpected syntax rules in const evaluator"); } diff --git a/crates/steel-core/src/steel_vm/engine.rs b/crates/steel-core/src/steel_vm/engine.rs index c0a4b015e..6db39a3bc 100644 --- a/crates/steel-core/src/steel_vm/engine.rs +++ b/crates/steel-core/src/steel_vm/engine.rs @@ -245,12 +245,25 @@ fn load_module_noop(target: &crate::rvals::SteelString) -> crate::rvals::Result< stop!(Generic => "This engine has not been given the capability to load dylibs") } +macro_rules! time { + ($target:expr, $label:expr, $e:expr) => {{ + let now = std::time::Instant::now(); + + let e = $e; + + log::debug!(target: $target, "{}: {:?}", $label, now.elapsed()); + + e + }}; +} + impl Engine { /// Function to access a kernel level execution environment /// Has access to primitives and syntax rules, but will not defer to a child /// kernel in the compiler pub(crate) fn new_kernel() -> Self { log::debug!(target:"kernel", "Instantiating a new kernel"); + let mut now = std::time::Instant::now(); let mut vm = Engine { virtual_machine: SteelThread::new(), @@ -262,12 +275,22 @@ impl Engine { dylibs: DylibContainers::new(), }; - register_builtin_modules(&mut vm); + time!( + "engine-creation", + "Registering builtin modules", + register_builtin_modules(&mut vm) + ); - vm.compile_and_run_raw_program(crate::steel_vm::primitives::ALL_MODULES) - .unwrap(); + time!( + "engine-creation", + "Loading the all modules prelude code", + vm.compile_and_run_raw_program(crate::steel_vm::primitives::ALL_MODULES) + .unwrap() + ); - log::debug!(target:"kernel", "Registered modules in the kernel!"); + log::debug!(target: "kernel", "Registered modules in the kernel!: {:?}", now.elapsed()); + + let mut now = std::time::Instant::now(); let core_libraries = [crate::stdlib::PRELUDE]; @@ -275,7 +298,7 @@ impl Engine { vm.compile_and_run_raw_program(core).unwrap(); } - log::debug!(target: "kernel", "Loaded prelude in the kernel!"); + log::debug!(target: "kernel", "Loaded prelude in the kernel!: {:?}", now.elapsed()); vm } @@ -316,6 +339,18 @@ impl Engine { return Engine::new_kernel(); } + if matches!(option_env!("STEEL_BOOTSTRAP"), Some("false") | None) { + let mut vm = Engine::new_kernel(); + + let sources = vm.sources.clone(); + + vm.register_fn("report-error!", move |error: SteelErr| { + raise_error(&sources, error); + }); + + return vm; + } + log::debug!(target:"kernel", "Instantiating a new kernel"); let mut vm = Engine { @@ -332,13 +367,10 @@ impl Engine { register_builtin_modules(&mut vm); for program in programs { - // println!("Running raw program..."); - vm.compiler.constant_map = program.constant_map.clone(); vm.virtual_machine.constant_map = program.constant_map.clone(); vm.run_raw_program(program).unwrap(); - // vm.run_raw_program_from_exprs(ast).unwrap(); } log::debug!(target: "kernel", "Loaded prelude in the kernel!"); @@ -1001,17 +1033,17 @@ impl Engine { pub fn new() -> Self { let mut engine = fresh_kernel_image(); - // Touch the printer to initialize it - // install_printer(); - // engine.register_fn("print-in-engine", print_in_engine); - engine.compiler.kernel = Some(Kernel::new()); + let now = std::time::Instant::now(); + if let Err(e) = engine.run(PRELUDE_WITHOUT_BASE) { raise_error(&engine.sources, e); panic!("This shouldn't happen!"); } + log::info!(target: "engine-creation", "Engine Creation: {:?}", now.elapsed()); + engine } @@ -1069,7 +1101,7 @@ impl Engine { // Registers the given module into the virtual machine pub fn register_module(&mut self, module: BuiltInModule) -> &mut Self { // Add the module to the map - self.modules.insert(Rc::clone(&module.name), module.clone()); + self.modules.insert(module.name(), module.clone()); // Register the actual module itself as a value to make the virtual machine capable of reading from it self.register_value( module.unreadable_name().as_str(), @@ -1087,7 +1119,7 @@ impl Engine { let external_module = FFIWrappedModule::new(module)?.build(); self.modules - .insert(external_module.name.clone(), external_module.clone()); + .insert(external_module.name(), external_module.clone()); self.register_value( external_module.unreadable_name().as_str(), @@ -1245,8 +1277,6 @@ impl Engine { &mut self.sources, )?; - // program.profile_instructions(); - self.run_raw_program(program) } diff --git a/crates/steel-core/src/steel_vm/primitives.rs b/crates/steel-core/src/steel_vm/primitives.rs index 4ac37cdc2..9cb18a878 100644 --- a/crates/steel-core/src/steel_vm/primitives.rs +++ b/crates/steel-core/src/steel_vm/primitives.rs @@ -158,7 +158,6 @@ const FIRST: &str = "first"; const REST: &str = "rest"; const APPEND: &str = "append"; const PUSH_BACK: &str = "push-back"; -const RANGE: &str = "range"; const LENGTH: &str = "length"; const REVERSE: &str = "reverse"; // const LIST_TO_VECTOR: &str = "list->vector"; @@ -197,8 +196,8 @@ pub const CONSTANTS: &[&str] = &[ "#%prim.first", REST, "#%prim.rest", - RANGE, - "#%prim.range", + // RANGE, + // "#%prim.range", NULL_HUH, "#%prim.null?", INT_HUH, @@ -498,6 +497,10 @@ pub static MODULE_IDENTIFIERS: Lazy> = Lazy::n set }); +// TODO: Make the prelude string generation lazy - so that +// the first time we load (steel/base) we don't have to regenerate +// the string. Probably just need a lazy static for loading 'steel/base' +// and then reference that directly. pub static ALL_MODULES: &str = r#" (require-builtin steel/hash) (require-builtin steel/sets) diff --git a/crates/steel-core/src/steel_vm/vm.rs b/crates/steel-core/src/steel_vm/vm.rs index f18abe95c..77936fc21 100644 --- a/crates/steel-core/src/steel_vm/vm.rs +++ b/crates/steel-core/src/steel_vm/vm.rs @@ -1137,6 +1137,10 @@ impl<'a> VmCore<'a> { ); } + fn weak_collection(&mut self) { + self.thread.heap.weak_collection(); + } + fn new_open_continuation_from_state(&self) -> Continuation { // println!("Creating new open continuation"); diff --git a/crates/steel-core/src/values/closed.rs b/crates/steel-core/src/values/closed.rs index 59060dc68..894a5ca93 100644 --- a/crates/steel-core/src/values/closed.rs +++ b/crates/steel-core/src/values/closed.rs @@ -281,6 +281,11 @@ impl Heap { self.vectors.len() } + pub fn weak_collection(&mut self) { + self.memory.retain(|x| Rc::weak_count(x) > 0); + self.vectors.retain(|x| Rc::weak_count(x) > 0); + } + // TODO: Call this in more areas in the VM to attempt to free memory more carefully // Also - come up with generational scheme if possible pub fn collect<'a>( @@ -418,20 +423,6 @@ impl Heap { let now = std::time::Instant::now(); - // println!("Freeing heap"); - - // TODO -> move destructors to another thread? - // That way the main thread is not blocked by the dropping of unreachable objects - - // println!( - // "Dropping memory: {:?}", - // self.memory - // .iter() - // .filter(|x| !x.borrow().is_reachable()) - // .map(|x| (Rc::weak_count(&x), x)) - // .collect::>() - // ); - log::debug!(target: "gc", "--- Sweeping ---"); let prior_len = self.memory.len() + self.vector_cells_allocated(); diff --git a/crates/steel-parser/src/ast.rs b/crates/steel-parser/src/ast.rs index 0018c5843..e71dc22b7 100644 --- a/crates/steel-parser/src/ast.rs +++ b/crates/steel-parser/src/ast.rs @@ -119,13 +119,21 @@ pub enum ExprKind { Begin(Begin), Return(Box), Quote(Box), - Macro(Macro), - SyntaxRules(SyntaxRules), + Macro(Box), + SyntaxRules(Box), List(List), Set(Box), Require(Require), } +#[test] +fn check_size() { + println!("ExprKind: {}", std::mem::size_of::()); + println!("SyntaxRules: {}", std::mem::size_of::()); + println!("Macro: {}", std::mem::size_of::()); + println!("List: {}", std::mem::size_of::()); +} + #[macro_export] macro_rules! expr_list { () => { $crate::ast::ExprKind::List($crate::ast::List::new(vec![])) }; @@ -925,6 +933,7 @@ impl From for ExprKind { pub struct List { pub args: Vec, pub syntax_object_id: usize, + pub improper: bool, } impl PartialEq for List { @@ -938,9 +947,15 @@ impl List { List { args, syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), + improper: false, } } + pub fn make_improper(mut self) -> Self { + self.improper = true; + self + } + pub fn is_empty(&self) -> bool { self.args.is_empty() } @@ -1192,7 +1207,7 @@ impl From for ExprKind { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Macro { pub name: Box, - pub syntax_rules: SyntaxRules, + pub syntax_rules: Box, pub location: SyntaxObject, } @@ -1216,7 +1231,7 @@ impl ToDoc for Macro { } impl Macro { - pub fn new(name: ExprKind, syntax_rules: SyntaxRules, location: SyntaxObject) -> Self { + pub fn new(name: ExprKind, syntax_rules: Box, location: SyntaxObject) -> Self { Macro { name: Box::new(name), syntax_rules, @@ -1227,7 +1242,7 @@ impl Macro { impl From for ExprKind { fn from(val: Macro) -> Self { - ExprKind::Macro(val) + ExprKind::Macro(Box::new(val)) } } @@ -1285,7 +1300,7 @@ impl ToDoc for SyntaxRules { impl From for ExprKind { fn from(val: SyntaxRules) -> Self { - ExprKind::SyntaxRules(val) + ExprKind::SyntaxRules(Box::new(val)) } } @@ -1909,7 +1924,11 @@ impl TryFrom> for ExprKind { )); }; - Ok(ExprKind::Macro(Macro::new(name, syntax_rules, syn))) + Ok(ExprKind::Macro(Box::new(Macro::new( + name, + syntax_rules, + syn, + )))) } TokenType::SyntaxRules => { let syn = a.syn.clone(); @@ -1957,9 +1976,9 @@ impl TryFrom> for ExprKind { } } - Ok(ExprKind::SyntaxRules(SyntaxRules::new( + Ok(ExprKind::SyntaxRules(Box::new(SyntaxRules::new( syntax_vec, pairs, syn, - ))) + )))) } _ => Ok(ExprKind::List(List::new(value))), } diff --git a/crates/steel-parser/src/parser.rs b/crates/steel-parser/src/parser.rs index 366817bf2..9d1eac89d 100644 --- a/crates/steel-parser/src/parser.rs +++ b/crates/steel-parser/src/parser.rs @@ -1200,11 +1200,11 @@ pub fn lower_macro_and_require_definitions(expr: ExprKind) -> Result { let name = value_iter.next().unwrap(); let syntax = lower_syntax_rules(value_iter.next().unwrap())?; - return Ok(ExprKind::Macro(Macro::new( + return Ok(ExprKind::Macro(Box::new(Macro::new( name, - syntax, + Box::new(syntax), define_syntax.into_atom_syntax_object().unwrap(), - ))); + )))); } if as_list.map(List::is_require).unwrap_or_default() { From f3ac3e72e8a02cf90135f8ae5feeae5847daa1b3 Mon Sep 17 00:00:00 2001 From: mattwparas Date: Sun, 10 Dec 2023 08:07:52 -0800 Subject: [PATCH 05/16] remove general const evaluation in interactive mode --- crates/steel-core/src/compiler/code_gen.rs | 5 +- crates/steel-core/src/compiler/compiler.rs | 38 ++++++----- crates/steel-core/src/compiler/modules.rs | 13 ++++ crates/steel-core/src/compiler/program.rs | 28 +++++--- crates/steel-core/src/core/instructions.rs | 21 ++++-- crates/steel-core/src/core/labels.rs | 44 +++++++++---- crates/steel-core/src/parser/expander.rs | 12 ++-- .../src/steel_vm/const_evaluation.rs | 4 +- crates/steel-core/src/steel_vm/engine.rs | 66 ++++++++++++++++++- r7rs-benchmarks/scheme.scm | 5 +- 10 files changed, 176 insertions(+), 60 deletions(-) diff --git a/crates/steel-core/src/compiler/code_gen.rs b/crates/steel-core/src/compiler/code_gen.rs index f52d4bff8..a80d16fb3 100644 --- a/crates/steel-core/src/compiler/code_gen.rs +++ b/crates/steel-core/src/compiler/code_gen.rs @@ -50,7 +50,6 @@ pub struct CodeGenerator<'a> { fn eval_atom(t: &SyntaxObject) -> Result { match &t.ty { TokenType::BooleanLiteral(b) => Ok((*b).into()), - // TokenType::Identifier(s) => env.borrow().lookup(&s), TokenType::NumberLiteral(n) => Ok(SteelVal::NumV(*n)), TokenType::StringLiteral(s) => Ok(SteelVal::StringV(s.into())), TokenType::CharacterLiteral(c) => Ok(SteelVal::CharV(*c)), @@ -642,6 +641,10 @@ impl<'a> VisitorMut for CodeGenerator<'a> { self.push( LabeledInstruction::builder(OpCode::PUSHCONST) .payload(idx) + // TODO: This is a little suspect, we're doing a bunch of stuff twice + // that we really don't need. In fact, we probably can get away with just... + // embedding the steel val directly here. + .list_contents(crate::parser::ast::ExprKind::Quote(Box::new(quote.clone()))) .constant(true), ); diff --git a/crates/steel-core/src/compiler/compiler.rs b/crates/steel-core/src/compiler/compiler.rs index 91da044c4..63d31b76d 100644 --- a/crates/steel-core/src/compiler/compiler.rs +++ b/crates/steel-core/src/compiler/compiler.rs @@ -9,6 +9,7 @@ use crate::{ reader::MultipleArityFunctions, shadow::RenameShadowedVariables, }, }, + core::labels::Expr, parser::{ ast::AstTools, expand_visitor::{expand_kernel, expand_kernel_in_env}, @@ -80,11 +81,11 @@ impl DebruijnIndicesInterner { Instruction { op_code: OpCode::BIND, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), span, .. - }), + })), .. }, Instruction { @@ -110,11 +111,11 @@ impl DebruijnIndicesInterner { Instruction { op_code: OpCode::BIND, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), span, .. - }), + })), .. }, .., @@ -176,10 +177,10 @@ impl DebruijnIndicesInterner { Instruction { op_code: OpCode::BIND, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), .. - }), + })), .. } => { // Keep track of where the defines actually are in the process @@ -188,21 +189,21 @@ impl DebruijnIndicesInterner { Instruction { op_code: OpCode::PUSH, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), span, .. - }), + })), .. } | Instruction { op_code: OpCode::SET, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), span, .. - }), + })), .. } => { if self.flat_defines.get(s).is_some() @@ -219,27 +220,26 @@ impl DebruijnIndicesInterner { // TODO commenting this for now if let Some(x) = instructions.get_mut(i) { x.payload_size = idx; - x.constant = false; } } Instruction { op_code: OpCode::CALLGLOBAL, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), span, .. - }), + })), .. } | Instruction { op_code: OpCode::CALLGLOBALTAIL, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(s), span, .. - }), + })), .. } => { if self.flat_defines.get(s).is_some() @@ -256,7 +256,6 @@ impl DebruijnIndicesInterner { // TODO commenting this for now if let Some(x) = instructions.get_mut(i) { x.payload_size = idx; - x.constant = false; } } _ => {} @@ -693,9 +692,11 @@ impl Compiler { semantic.replace_anonymous_function_calls_with_plain_lets(); + Ok(expanded_statements) + // Done lowering anonymous function calls to let - self.apply_const_evaluation(constants, expanded_statements, true) + // self.apply_const_evaluation(constants, expanded_statements, true) } // TODO @@ -729,6 +730,9 @@ impl Compiler { // Make sure to apply the peephole optimizations raw_program.apply_optimizations(); + // Lets see everything that gets run! + // raw_program.debug_print(); + Ok(raw_program) } diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index 4951afdcd..7a7fab58d 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -659,6 +659,10 @@ impl ModuleManager { } } +// Pre-compile module to bytecode? Is it even possible? +// Dynamically linking the module would then make it relatively +// easy to just load everything up at the start. +// Compiled module _should_ be possible now. Just create a target #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct CompiledModule { name: PathBuf, @@ -1661,9 +1665,18 @@ impl<'a> ModuleBuilder<'a> { module.set_emitted(true); + // println!( + // "-------------- Emitting module: {:?} ----------------------", + // self.name + // ); + let mut result = module.to_top_level_module(self.compiled_modules, self.global_macro_map)?; + // println!("{}", result.to_pretty(60)); + + // println!("------------------ Finish ----------------------------------"); + // let mut analysis = Analysis::from_exprs(&[result]); // let mut semantic = SemanticAnalysis::from_analysis(&mut result, analysis); diff --git a/crates/steel-core/src/compiler/program.rs b/crates/steel-core/src/compiler/program.rs index 96f217ad3..5edeb82c0 100644 --- a/crates/steel-core/src/compiler/program.rs +++ b/crates/steel-core/src/compiler/program.rs @@ -1,3 +1,5 @@ +use crate::core::labels::Expr; +use crate::parser::span_visitor::get_span; use crate::rvals::Result; use crate::{ compiler::constants::ConstantMap, @@ -106,15 +108,15 @@ pub fn specialize_constants(instructions: &mut [Instruction]) -> Result<()> { Some(Instruction { op_code: OpCode::PUSHCONST, contents: - Some(SyntaxObject { + Some(Expr::Atom(SyntaxObject { ty: TokenType::Identifier(_), .. - }), + })), .. }) => continue, Some(Instruction { op_code: OpCode::PUSHCONST, - contents: Some(syn), + contents: Some(Expr::Atom(syn)), .. }) => { let value = eval_atom(syn)?; @@ -149,7 +151,7 @@ pub fn convert_call_globals(instructions: &mut [Instruction]) { Some(Instruction { op_code: OpCode::PUSH, payload_size: index, - contents: Some(ident), + contents: Some(Expr::Atom(ident)), .. }), Some(Instruction { @@ -237,7 +239,7 @@ pub fn convert_call_globals(instructions: &mut [Instruction]) { Some(Instruction { op_code: OpCode::PUSH, payload_size: index, - contents: Some(ident), + contents: Some(Expr::Atom(ident)), .. }), Some(Instruction { @@ -405,10 +407,10 @@ pub fn inline_num_operations(instructions: &mut [Instruction]) { Some(Instruction { op_code: OpCode::FUNC | OpCode::TAILCALL, contents: - Some(RawSyntaxObject { + Some(Expr::Atom(RawSyntaxObject { ty: TokenType::Identifier(ident), .. - }), + })), payload_size, .. }), @@ -645,7 +647,7 @@ impl Program { // This way, the VM knows where to look up values #[derive(Clone)] pub struct RawProgramWithSymbols { - instructions: Vec>, + pub(crate) instructions: Vec>, pub(crate) constant_map: ConstantMap, version: String, // TODO -> this should be semver } @@ -1041,7 +1043,15 @@ fn extract_spans( .iter() .map(|x| { x.iter() - .map(|x| x.contents.as_ref().map(|x| x.span).unwrap_or_default()) + .map(|x| { + x.contents + .as_ref() + .map(|x| match x { + Expr::Atom(a) => a.span, + Expr::List(l) => get_span(l), + }) + .unwrap_or_default() + }) .collect() }) .collect(); diff --git a/crates/steel-core/src/core/instructions.rs b/crates/steel-core/src/core/instructions.rs index 3a991a1f3..159e39f14 100644 --- a/crates/steel-core/src/core/instructions.rs +++ b/crates/steel-core/src/core/instructions.rs @@ -1,8 +1,9 @@ use crate::core::opcode::OpCode; -use crate::parser::parser::SyntaxObject; use serde::{Deserialize, Serialize}; use std::convert::TryInto; +use super::labels::Expr; + /// Instruction loaded with lots of information prior to being condensed /// Includes the opcode and the payload size, plus some information /// used for locating spans and pretty error messages @@ -10,21 +11,19 @@ use std::convert::TryInto; pub struct Instruction { pub op_code: OpCode, pub payload_size: usize, - pub contents: Option, - pub constant: bool, + pub contents: Option, } impl Instruction { pub fn new_from_parts( op_code: OpCode, payload_size: usize, - contents: Option, + contents: Option, ) -> Instruction { Instruction { op_code, payload_size, contents, - constant: false, } } } @@ -96,8 +95,16 @@ pub fn disassemble(instructions: &[Instruction]) -> String { buffer.push_str(" "); if let Some(syn) = instruction.contents.as_ref() { - let contents = syn.ty.to_string(); - buffer.push_str(contents.as_str()); + match syn { + Expr::Atom(syn) => { + let contents = syn.ty.to_string(); + buffer.push_str(contents.as_str()); + } + Expr::List(l) => { + let contents = l.to_string(); + buffer.push_str(contents.as_str()); + } + } } buffer.push('\n'); diff --git a/crates/steel-core/src/core/labels.rs b/crates/steel-core/src/core/labels.rs index 77c3f85c8..e44ada703 100644 --- a/crates/steel-core/src/core/labels.rs +++ b/crates/steel-core/src/core/labels.rs @@ -1,3 +1,6 @@ +use serde::{Deserialize, Serialize}; +use steel_parser::ast::ExprKind; + use super::instructions::Instruction; use super::opcode::OpCode; use crate::parser::parser::SyntaxObject; @@ -16,11 +19,17 @@ pub fn fresh() -> Label { Label(LABEL_ID.fetch_add(1, Ordering::Relaxed)) } +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] +pub enum Expr { + Atom(SyntaxObject), + List(ExprKind), +} + #[derive(Clone, Debug)] pub struct LabeledInstruction { pub op_code: OpCode, pub payload_size: usize, - pub contents: Option, + pub contents: Option, pub tag: Option