Detect and auto complete "nullable" productions (#113)

* improve `AppendOnlyVec` to use it in more places
* separate "term matching" from "state"
* rename `State` to `Traversal` because it was confusing me (this may be controversial! please let's talk about it!)
* detect "nullable" productions, and autocomplete them during parsing
* added "tracing" !

.github/actions-rs/grcov.yml

@@ -5,3 +5,5 @@ filter: covered
 output-type: lcov
 output-path: ./lcov.info
 prefix-dir: /home/user/build/
+ignore:
+  - "../*"

Cargo.toml

@@ -14,10 +14,10 @@ repository = "https://github.com/shnewto/bnf"
 
 license = "MIT"
 
-[features]
-unstable = []
-
 [dependencies]
+tracing = { version = "0.1.37", optional = true }
+tracing-subscriber = { version = "0.3.16", optional = true }
+tracing-flame = { version = "0.2.0", optional = true }
 
 [dependencies.stacker]
 version = "0.1.2"
@@ -41,6 +41,11 @@ version = "1.0.2"
 [dev-dependencies]
 criterion = "0.3.5"
 
+[features]
+default = []
+unstable = []
+tracing = ["dep:tracing", "dep:tracing-subscriber", "dep:tracing-flame"]
+
 [[bench]]
 name = "bnf"
 harness = false

benches/README.md

@@ -3,6 +3,7 @@
 Benchmarking numbers will vary across tests, specific grammars, rust versions, and hardware. With so many sources of noise, it is important to remember that "faster" is not always easy to define.
 
 With that in mind, BNF's benchmarking has the following goals:
+
 * identify statistically significant performance regressions
 * validate performance hypothesis
 
@@ -29,4 +30,18 @@ These benchmarks are not run during continuous integration testing. But if a dev
 > cargo criterion
 
 [criterion]: https://crates.io/crates/criterion
-[cargo-criterion]: https://github.com/bheisler/cargo-criterion
+[cargo-criterion]: https://github.com/bheisler/cargo-criterion
+
+#### Flamegraph
+
+> CARGO_PROFILE_BENCH_DEBUG=true cargo flamegraph --bench bnf -- --bench
+
+`sudo` may be required for `dtrace` on macOS
+
+#### Tracing
+
+BNF has an optional "tracing" feature which will provide tracing spans during parsing.
+
+The benchmarks are enabled to write these tracing spans to `tracing.folded`. This data can then be parsed to provide a flamegraph.
+
+> RUST_LOG=TRACE cargo criterion --features "tracing" && cat tracing.folded | inferno-flamegraph > flamegraph.svg

benches/bnf.rs

@@ -2,7 +2,31 @@ use bnf::Grammar;
 use criterion::{criterion_group, criterion_main, Criterion};
 use rand::seq::SliceRandom;
 
+#[cfg(feature = "tracing")]
+fn init_tracing() -> impl Drop {
+    use tracing_flame::FlameLayer;
+    use tracing_subscriber::{fmt, prelude::*};
+    let fmt_layer = fmt::Layer::default();
+
+    let (flame_layer, _guard) = FlameLayer::with_file("./tracing.folded").unwrap();
+
+    tracing_subscriber::registry()
+        .with(fmt_layer)
+        .with(flame_layer)
+        .init();
+
+    _guard
+}
+
+#[cfg(not(feature = "tracing"))]
+fn init_tracing() {}
+
 fn examples(c: &mut Criterion) {
+    let _tracing = init_tracing();
+
+    #[cfg(feature = "tracing")]
+    let _span = tracing::span!(tracing::Level::TRACE, "BENCH ITER").entered();
+
     c.bench_function("parse postal", |b| {
         let input = std::include_str!("../tests/fixtures/postal_address.terminated.input.bnf");
         b.iter(|| input.parse::<Grammar>().unwrap());

src/append_vec.rs

@@ -0,0 +1,104 @@
+/// Create a new id type for an [`AppendOnlyVec`], which will be a wrapped [`usize`].
+/// Example usage: `append_only_vec_id!(pub(crate) ProductionId)`;
+macro_rules! append_only_vec_id {
+    ($visible:vis $id:ident) => {
+        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+        $visible struct $id(usize);
+
+        impl From<usize> for $id {
+            fn from(id: usize) -> Self {
+                Self(id)
+            }
+        }
+
+        impl From<$id> for usize {
+            fn from(id: $id) -> Self {
+                id.0
+            }
+        }
+    };
+}
+
+pub(crate) use append_only_vec_id;
+
+/// Vector type which does *not* allow item removal during lifetime.
+/// Useful for data structures with complex, shared ownership, such as graphs.
+#[derive(Debug, Clone)]
+pub(crate) struct AppendOnlyVec<T, I> {
+    vec: Vec<T>,
+    id_type: std::marker::PhantomData<I>,
+}
+
+impl<T, I> AppendOnlyVec<T, I>
+where
+    I: From<usize> + Into<usize>,
+{
+    pub fn new() -> Self {
+        Self::default()
+    }
+    pub fn len(&self) -> usize {
+        self.vec.len()
+    }
+    fn next_id(&self) -> I {
+        I::from(self.len())
+    }
+    pub fn push(&mut self, item: T) -> I {
+        let id = self.next_id();
+        self.vec.push(item);
+        id
+    }
+    pub fn push_with_id<F>(&mut self, build: F) -> &T
+    where
+        F: Fn(I) -> T,
+    {
+        let id = self.next_id();
+        let item = build(id);
+        let id = self.push(item);
+        self.get(id).expect("failed to get appended item")
+    }
+    pub fn get(&self, id: I) -> Option<&T> {
+        self.vec.get::<usize>(id.into())
+    }
+    pub fn iter(&self) -> impl Iterator<Item = &T> {
+        self.vec.iter()
+    }
+}
+
+impl<T, K> Default for AppendOnlyVec<T, K> {
+    fn default() -> Self {
+        Self::from(vec![])
+    }
+}
+
+impl<T, K> From<Vec<T>> for AppendOnlyVec<T, K> {
+    fn from(vec: Vec<T>) -> Self {
+        Self {
+            vec,
+            id_type: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<T, K> IntoIterator for AppendOnlyVec<T, K> {
+    type Item = <Vec<T> as IntoIterator>::Item;
+    type IntoIter = <Vec<T> as IntoIterator>::IntoIter;
+    fn into_iter(self) -> Self::IntoIter {
+        self.vec.into_iter()
+    }
+}
+
+impl<'a, T, K> IntoIterator for &'a AppendOnlyVec<T, K> {
+    type Item = &'a T;
+    type IntoIter = std::slice::Iter<'a, T>;
+    fn into_iter(self) -> Self::IntoIter {
+        self.vec.iter()
+    }
+}
+
+impl<'a, T, K> IntoIterator for &'a mut AppendOnlyVec<T, K> {
+    type Item = &'a mut T;
+    type IntoIter = std::slice::IterMut<'a, T>;
+    fn into_iter(self) -> Self::IntoIter {
+        self.vec.iter_mut()
+    }
+}