feat(note-transport): add nonce to FetchNotes cursor

bin/note-transport/Cargo.toml

@@ -11,6 +11,10 @@ repository.workspace   = true
 rust-version.workspace = true
 version.workspace      = true
 
+[package.metadata.cargo-shear]
+# The build script links migrations through the generated migrator.
+ignored-paths = ["src/db/migrations/*.rs"]
+
 [lints]
 workspace = true
 
@@ -30,6 +34,7 @@ miden-node-tracing     = { workspace = true }
 miden-node-utils       = { workspace = true }
 miden-protocol         = { workspace = true }
 prost                  = { workspace = true }
+rand                   = { workspace = true }
 rusqlite               = { workspace = true }
 thiserror              = { workspace = true }
 tokio                  = { features = ["macros", "net", "rt-multi-thread"], workspace = true }

bin/note-transport/README.md

@@ -46,23 +46,37 @@ without chain lookup; an absent hint differs from block zero.
 A retry with the same note ID succeeds and keeps the first envelope, timestamp, and cursor. This also applies when the
 retry supplies a different block hint or storage is full.
 
-`FetchNotes` accepts at most 128 tags and an exclusive cursor. Start with cursor zero. Use each response cursor for the
-next request with the same set of tags. Results follow insertion order across all requested tags. Duplicate tags do not
-duplicate results. Empty pages retain the request cursor. The response `has_more` field indicates that another page is
-available.
+`FetchNotes` accepts at most 128 tags and an exclusive cursor with a `fixed64` database nonce and a `fixed64` sequence.
+Omit the cursor to start from the first retained note. Store the complete response cursor and use it for the next
+request with the same set of tags. Results follow insertion order across all requested tags. Duplicate tags do not
+duplicate results. Every successful response includes a cursor. An initial empty page returns the current nonce and
+sequence zero. Later empty pages retain the request cursor. The response `has_more` field indicates that another page is
+available. Nonce zero is valid and has no special meaning.
 
-A cursor belongs to the requested set of tags. Reset the cursor to zero when you add or remove tags. Reordering tags or
-changing duplicate tags does not change the set. A restart can return notes that you fetched before. Use note IDs to
-remove duplicate results.
+A cursor belongs to the requested set of tags. Clear the cursor when you add or remove tags. Reordering tags or changing
+duplicate tags does not change the set. Restarting a fetch from the beginning can return notes that you fetched before.
+Use note IDs to remove duplicate results.
 
-For example, after you fetch tag A through cursor 100, reset the cursor to zero when you add tag B. If you reuse cursor
-100, you skip retained notes for tag B with cursors at or below 100.
+For example, after you fetch tag A through cursor 100, clear the cursor when you add tag B. If you reuse cursor 100, you
+skip retained notes for tag B with cursors at or below 100.
 
 A page contains at most 500 notes and 3 MiB of canonically serialized header and detail bytes. The encoded response also
 fits the default 4 MiB gRPC client limit. The default per-note limit is 512,000 bytes. `--max-note-size` can change it
 up to the page limit. The required `--max-storage-bytes` limits retained header and detail bytes. It does not include
-SQLite indexes, database metadata, or WAL disk usage. Cursors use the positive signed 64-bit range supported by SQLite.
-Recipients must poll before notes expire.
+SQLite indexes, database metadata, or WAL disk usage. Cursor sequences use the positive signed 64-bit range supported by
+SQLite. Sequence zero starts a fetch. Nonces use the complete unsigned 64-bit range. Recipients must poll before notes
+expire.
+
+The service generates a random nonce when it creates the database. Schema migration initializes the nonce for existing
+databases. Ordinary service restarts, repeated migrations, and retention cleanup preserve the nonce. A cursor from a
+different database generation returns `FAILED_PRECONDITION`. Clear the cursor and fetch again after this error.
+Deduplicate results by note ID. The nonce detects a generation change; it does not recover lost notes or authenticate
+cursors.
+
+The structured cursor is incompatible with the scalar cursor API. Coordinate client and server upgrades. Discard
+persisted scalar cursors when upgrading clients. Run the database migration before starting the updated service.
+Restoring an older backup also restores its nonce. That recovery procedure must rotate the nonce before the service
+starts. This service does not provide a nonce rotation command.
 
 Malformed requests return `INVALID_ARGUMENT`. Note size and storage capacity limits return `RESOURCE_EXHAUSTED`. Storage
 failures return `INTERNAL` and are logged by the service.

bin/note-transport/src/db/migrations/002_cursor_nonce.rs

@@ -0,0 +1,23 @@
+use rusqlite::Transaction;
+
+/// Assigns a database nonce without changing notes or storage counters.
+pub fn migrate(tx: &Transaction<'_>) -> anyhow::Result<()> {
+    tx.execute_batch(
+        "CREATE TABLE storage_metadata_with_nonce (
+            singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
+            retained_bytes INTEGER NOT NULL CHECK (retained_bytes >= 0),
+            nonce BLOB NOT NULL CHECK (length(nonce) = 8)
+        ) STRICT;",
+    )?;
+    let nonce = rand::random::<u64>().to_le_bytes();
+    tx.execute(
+        "INSERT INTO storage_metadata_with_nonce (singleton, retained_bytes, nonce)
+         SELECT singleton, retained_bytes, ?1 FROM storage_metadata",
+        [nonce.as_slice()],
+    )?;
+    tx.execute_batch(
+        "DROP TABLE storage_metadata;
+         ALTER TABLE storage_metadata_with_nonce RENAME TO storage_metadata;",
+    )?;
+    Ok(())
+}

bin/note-transport/src/db/mod.rs

@@ -52,12 +52,22 @@ pub enum StorageError {
     Capacity(String),
     #[error("invalid cursor")]
     InvalidCursor,
+    #[error("cursor belongs to another database generation; clear the cursor and retry")]
+    StaleCursor,
     #[error("{0}")]
     InvalidData(String),
 }
 
+/// A position in one database generation.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Cursor {
+    pub nonce: u64,
+    pub sequence: u64,
+}
+
 #[derive(Debug)]
 pub struct FetchPage {
+    pub cursor: Cursor,
     pub notes: Vec<StoredNote>,
     pub has_more: bool,
 }
@@ -134,7 +144,7 @@ pub async fn store_note(
                     "accepting this note would exceed the {max_retained_bytes} byte limit"
                 )));
             }
-            queries::update_storage_metadata(tx, next_retained)?;
+            queries::update_retained_bytes(tx, next_retained)?;
             Ok(StoreResult::Inserted)
         })
         .await
@@ -145,15 +155,17 @@ pub async fn store_note(
 pub async fn fetch_notes(
     reader: &DbReader,
     tags: Vec<u32>,
-    cursor: u64,
+    cursor: Option<Cursor>,
 ) -> Result<FetchPage, StorageError> {
-    let cursor = i64::try_from(cursor).map_err(|_| StorageError::InvalidCursor)?;
-    if tags.is_empty() {
-        return Ok(FetchPage { notes: vec![], has_more: false });
-    }
+    let sequence = cursor.map_or(0, |cursor| cursor.sequence);
+    let sequence = i64::try_from(sequence).map_err(|_| StorageError::InvalidCursor)?;
     reader
         .read("fetch_notes", move |tx| {
-            queries::fetch_notes(tx, tags.into_iter().map(NoteTag::new).collect(), cursor)
+            let nonce = queries::select_nonce(tx)?;
+            if cursor.is_some_and(|cursor| cursor.nonce != nonce) {
+                return Err(StorageError::StaleCursor);
+            }
+            queries::fetch_notes(tx, tags.into_iter().map(NoteTag::new).collect(), sequence, nonce)
         })
         .await
 }

bin/note-transport/src/db/queries/fetch_notes/mod.rs

@@ -1,13 +1,21 @@
 use miden_node_db::sqlite::{InList, ReadTx};
 use miden_protocol::note::NoteTag;
 
-use crate::db::{FETCH_NOTES_MAX_BYTES, FETCH_NOTES_MAX_ROWS, FetchPage, StorageError, StoredNote};
+use crate::db::{
+    Cursor,
+    FETCH_NOTES_MAX_BYTES,
+    FETCH_NOTES_MAX_ROWS,
+    FetchPage,
+    StorageError,
+    StoredNote,
+};
 
 /// Returns a page in cursor order. Row and byte limits apply before note blobs are loaded.
 pub fn fetch_notes(
     tx: &ReadTx<'_>,
     tags: Vec<NoteTag>,
     cursor: i64,
+    nonce: u64,
 ) -> Result<FetchPage, StorageError> {
     let tags = InList::from_values(tags);
     let rows = tx.query(
@@ -35,5 +43,12 @@ pub fn fetch_notes(
     let candidate_count = rows.first().map_or(0, |(_, count)| *count);
     let notes: Vec<_> = rows.into_iter().map(|(note, _)| note).collect();
     let has_more = candidate_count > i64::try_from(notes.len()).expect("page length fits i64");
-    Ok(FetchPage { notes, has_more })
+    let sequence = notes.last().map_or(cursor, |note| note.seq);
+    let sequence = u64::try_from(sequence)
+        .map_err(|_| StorageError::InvalidData("invalid stored note sequence".into()))?;
+    Ok(FetchPage {
+        notes,
+        has_more,
+        cursor: Cursor { nonce, sequence },
+    })
 }

bin/note-transport/src/db/queries/mod.rs

@@ -6,12 +6,15 @@ pub use note_exists::note_exists;
 mod insert_note;
 pub use insert_note::insert_note;
 
-mod update_storage_metadata;
-pub use update_storage_metadata::update_storage_metadata;
+mod update_retained_bytes;
+pub use update_retained_bytes::update_retained_bytes;
 
 mod select_retained_bytes;
 pub use select_retained_bytes::select_retained_bytes;
 
+mod select_nonce;
+pub use select_nonce::select_nonce;
+
 mod delete_notes_created_before;
 pub use delete_notes_created_before::delete_notes_created_before;
 

bin/note-transport/src/db/queries/select_nonce/mod.rs

@@ -0,0 +1,17 @@
+use miden_node_db::sqlite::ReadTx;
+
+use crate::db::StorageError;
+
+/// Reads the retained payload size.
+pub fn select_nonce(tx: &ReadTx<'_>) -> Result<u64, StorageError> {
+    let nonce = tx
+        .query(include_str!("select_nonce.sql"), &[], |row| row.get::<Vec<u8>>(0))?
+        .into_iter()
+        .next()
+        .ok_or_else(|| StorageError::InvalidData("storage metadata is missing".into()))?;
+
+    nonce
+        .try_into()
+        .map_err(|_| StorageError::InvalidData("invalid database nonce length".into()))
+        .map(u64::from_le_bytes)
+}

bin/note-transport/src/db/queries/select_nonce/select_nonce.sql

@@ -0,0 +1,3 @@
+SELECT nonce
+FROM storage_metadata
+WHERE singleton = 1;

bin/note-transport/src/db/queries/update_storage_metadata/mod.rs → bin/note-transport/src/db/queries/update_retained_bytes/mod.rs

@@ -2,7 +2,7 @@ use miden_node_db::DatabaseError;
 use miden_node_db::sqlite::WriteTx;
 
 /// Updates the retained payload size in the current transaction.
-pub fn update_storage_metadata(tx: &WriteTx<'_>, retained_bytes: i64) -> Result<(), DatabaseError> {
+pub fn update_retained_bytes(tx: &WriteTx<'_>, retained_bytes: i64) -> Result<(), DatabaseError> {
     tx.execute(include_str!("update_storage_metadata.sql"), &[&retained_bytes])?;
     Ok(())
 }