Implement :from filter

docs/src/reference/filters.md

@@ -114,13 +114,34 @@ commits that don't match any of the other shas.
 Produce the history that would be the result of pushing the passed branches with the
 passed filters into the upstream.
 
+### Start filtering from a specific commit **:from(<sha>:filter)**
+
+Produce a history that keeps the original history leading up to the specified commit `<sha>` unchanged,
+but applies the given `:filter` to all commits from that commit onwards.
+
 ### Prune trivial merge commits **:prune=trivial-merge**
 
 Produce a history that skips all merge commits whose tree is identical to the first parents
 tree.
 Normally Josh will keep all commits in the filtered history whose tree differs from any of it's
 parents.
 
+### Pin tree contents
+
+`:pin` filter prevents appearance of selected subtrees for a given revision.
+
+In practical terms, it means that file and folder updates are "held off", and revisions are "pinned".
+If a tree entry already existed in the parent revision, that version will be chosen.
+Otherwise, the tree entry will not appear in the filtered commit.
+
+The source of the parent revision is always the first commit parent.
+
+Note that this filter is only practical when used with `:hook` or `workspace.josh`,
+as it should apply per-revision only. Applying `:pin` for the whole history
+will result in the subtree being excluded from all revisions.
+
+Refer to `pin_filter_workspace.t` and `pin_filter_hook.t` for reference.
+
 Filter order matters
 --------------------
 

josh-core/src/cache.rs

@@ -9,9 +9,20 @@ use std::sync::{LazyLock, RwLock};
 pub(crate) const CACHE_VERSION: u64 = 24;
 
 pub trait CacheBackend: Send + Sync {
-    fn read(&self, filter: filter::Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>>;
-
-    fn write(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()>;
+    fn read(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>>;
+
+    fn write(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()>;
 }
 
 pub trait FilterHook {
@@ -323,6 +334,11 @@ impl Transaction {
     }
 
     pub fn insert(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid, store: bool) {
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).unwrap()
+        } else {
+            0
+        };
         let mut t2 = self.t2.borrow_mut();
         t2.commit_map
             .entry(filter.id())
@@ -334,14 +350,13 @@ impl Transaction {
         // the history length by a very large factor.
         if store || from.as_bytes()[0] == 0 {
             t2.cache
-                .write_all(filter, from, to)
+                .write_all(filter, from, to, sequence_number)
                 .expect("Failed to write cache");
         }
     }
 
     pub fn get_missing(&self) -> Vec<(filter::Filter, git2::Oid)> {
         let mut missing = self.t2.borrow().missing.clone();
-        missing.sort_by_key(|(f, i)| (filter::nesting(*f), *f, *i));
         missing.dedup();
         missing.retain(|(f, i)| !self.known(*f, *i));
         self.t2.borrow_mut().missing = missing.clone();
@@ -358,7 +373,9 @@ impl Transaction {
         } else {
             let mut t2 = self.t2.borrow_mut();
             t2.misses += 1;
-            t2.missing.push((filter, from));
+            if !t2.missing.contains(&(filter, from)) {
+                t2.missing.insert(0, (filter, from));
+            }
             None
         }
     }
@@ -367,6 +384,11 @@ impl Transaction {
         if filter == filter::nop() {
             return Some(from);
         }
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).unwrap()
+        } else {
+            0
+        };
         let t2 = self.t2.borrow_mut();
         if let Some(m) = t2.commit_map.get(&filter.id()) {
             if let Some(oid) = m.get(&from).cloned() {
@@ -376,7 +398,7 @@ impl Transaction {
 
         let oid = t2
             .cache
-            .read_propagate(filter, from)
+            .read_propagate(filter, from, sequence_number)
             .expect("Failed to read from cache backend");
 
         let oid = if let Some(oid) = oid { Some(oid) } else { None };
@@ -385,6 +407,9 @@ impl Transaction {
             if oid == git2::Oid::zero() {
                 return Some(oid);
             }
+            if filter == filter::sequence_number() {
+                return Some(oid);
+            }
 
             if self.repo.odb().unwrap().exists(oid) {
                 // Only report an object as cached if it exists in the object database.
@@ -396,3 +421,70 @@ impl Transaction {
         None
     }
 }
+
+/// Encode a `u128` into a 20-byte git OID (SHA-1 sized).
+/// The high 4 bytes of the OID are zero; the low 16 bytes
+/// contain the big-endian integer.
+pub fn oid_from_u128(n: u128) -> git2::Oid {
+    let mut bytes = [0u8; 20];
+    // place the 16 integer bytes at the end (big-endian)
+    bytes[20 - 16..].copy_from_slice(&n.to_be_bytes());
+    // Safe: length is exactly 20
+    git2::Oid::from_bytes(&bytes).expect("20-byte OID construction cannot fail")
+}
+
+/// Decode a `u128` previously encoded by `oid_from_u128`.
+pub fn u128_from_oid(oid: git2::Oid) -> u128 {
+    let b = oid.as_bytes();
+    let mut n = [0u8; 16];
+    n.copy_from_slice(&b[20 - 16..]); // take the last 16 bytes
+    u128::from_be_bytes(n)
+}
+
+pub fn compute_sequence_number(
+    transaction: &cache::Transaction,
+    input: git2::Oid,
+) -> JoshResult<u128> {
+    if let Some(count) = transaction.get(filter::sequence_number(), input) {
+        return Ok(u128_from_oid(count));
+    }
+
+    let commit = transaction.repo().find_commit(input)?;
+    if let Some(p) = commit.parent_ids().next() {
+        if let Some(count) = transaction.get(filter::sequence_number(), p) {
+            let pc = u128_from_oid(count);
+            transaction.insert(
+                filter::sequence_number(),
+                input,
+                oid_from_u128(pc + 1),
+                true,
+            );
+            return Ok(pc + 1);
+        }
+    }
+
+    let mut walk = transaction.repo().revwalk()?;
+    walk.set_sorting(git2::Sort::REVERSE | git2::Sort::TOPOLOGICAL)?;
+    walk.push(input)?;
+
+    for c in walk {
+        let commit = transaction.repo().find_commit(c?)?;
+        let pc = if let Some(p) = commit.parent_ids().next() {
+            compute_sequence_number(transaction, p)?
+        } else {
+            0
+        };
+
+        transaction.insert(
+            filter::sequence_number(),
+            commit.id(),
+            oid_from_u128(pc + 1),
+            true,
+        );
+    }
+    if let Some(count) = transaction.get(filter::sequence_number(), input) {
+        Ok(u128_from_oid(count))
+    } else {
+        Err(josh_error("missing sequence_number"))
+    }
+}

josh-core/src/cache_notes.rs

@@ -1,5 +1,6 @@
 use crate::JoshResult;
 use crate::cache::{CACHE_VERSION, CacheBackend};
+use crate::filter;
 use crate::filter::Filter;
 
 pub struct NotesCacheBackend {
@@ -15,24 +16,43 @@ impl NotesCacheBackend {
     }
 }
 
-fn is_note_eligible(oid: git2::Oid) -> bool {
-    oid.as_bytes()[0] == 0
+fn is_note_eligible(repo: &git2::Repository, oid: git2::Oid, sequence_number: u128) -> bool {
+    let parent_count = if let Ok(c) = repo.find_commit(oid) {
+        c.parent_ids().count()
+    } else {
+        return false;
+    };
+
+    sequence_number % 100 == 0 || parent_count != 1
 }
 
-fn note_path(key: git2::Oid) -> String {
-    format!("refs/josh/{}/{}", CACHE_VERSION, key)
+fn note_path(key: git2::Oid, sequence_number: u128) -> String {
+    format!(
+        "refs/josh/{}/{}/{}",
+        CACHE_VERSION,
+        sequence_number / 10000,
+        key,
+    )
 }
 
 impl CacheBackend for NotesCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
+        if filter == filter::sequence_number() {
+            return Ok(None);
+        }
         let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
-
-        if !is_note_eligible(from) {
+        if !is_note_eligible(&repo, from, sequence_number) {
             return Ok(None);
         }
 
-        if let Ok(note) = repo.find_note(Some(&note_path(key)), from) {
+        let key = crate::filter::as_tree(&*repo, filter)?;
+
+        if let Ok(note) = repo.find_note(Some(&note_path(key, sequence_number)), from) {
             let message = note.message().unwrap_or("");
             let result = git2::Oid::from_str(message)?;
 
@@ -42,20 +62,29 @@ impl CacheBackend for NotesCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
-        let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()> {
+        if filter == filter::sequence_number() {
+            return Ok(());
+        }
 
-        if !is_note_eligible(from) {
+        let repo = self.repo.lock()?;
+        if !is_note_eligible(&*repo, from, sequence_number) {
             return Ok(());
         }
 
+        let key = crate::filter::as_tree(&*repo, filter)?;
         let signature = crate::cache::josh_commit_signature()?;
 
         repo.note(
             &signature,
             &signature,
-            Some(&note_path(key)),
+            Some(&note_path(key, sequence_number)),
             from,
             &to.to_string(),
             true,

josh-core/src/cache_sled.rs

@@ -80,7 +80,12 @@ fn insert_sled_tree(filter: Filter) -> sled::Tree {
 }
 
 impl CacheBackend for SledCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())
@@ -94,7 +99,13 @@ impl CacheBackend for SledCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<()> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())

josh-core/src/cache_stack.rs

@@ -33,9 +33,10 @@ impl CacheStack {
         filter: filter::Filter,
         from: git2::Oid,
         to: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<()> {
         for backend in &self.backends {
-            backend.write(filter, from, to)?;
+            backend.write(filter, from, to, sequence_number)?;
         }
 
         Ok(())
@@ -51,16 +52,19 @@ impl CacheStack {
         &self,
         filter: filter::Filter,
         from: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<Option<git2::Oid>> {
         let values = self
             .backends
             .iter()
             .enumerate()
-            .find_map(|(index, backend)| match backend.read(filter, from) {
-                Ok(None) => None,
-                Ok(Some(oid)) => Some(Ok((index, oid))),
-                Err(e) => Some(Err(e)),
-            });
+            .find_map(
+                |(index, backend)| match backend.read(filter, from, sequence_number) {
+                    Ok(None) => None,
+                    Ok(Some(oid)) => Some(Ok((index, oid))),
+                    Err(e) => Some(Err(e)),
+                },
+            );
 
         let (index, oid) = match values {
             // None of the backends had the value
@@ -74,7 +78,7 @@ impl CacheStack {
         self.backends
             .iter()
             .take(index)
-            .try_for_each(|backend| backend.write(filter, from, oid))?;
+            .try_for_each(|backend| backend.write(filter, from, oid, sequence_number))?;
 
         Ok(Some(oid))
     }