From 4e8a51b808d61633da4dd1ededaa4724e232b65d Mon Sep 17 00:00:00 2001
From: Jun Matsushita <jun@iilab.org>
Date: Mon, 27 Sep 2021 18:18:43 +0200
Subject: [PATCH 1/3] Guard against null node in history. Fixes #46

---
 index.js       | 2 +-
 lib/history.js | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/index.js b/index.js
index 980c408..b6acb99 100644
--- a/index.js
+++ b/index.js
@@ -330,7 +330,7 @@ HyperTrie.prototype.getBySeq = function (seq, opts, cb) {
     const node = Node.decode(val, seq, self.valueEncoding, self.hash)
     self._cache.set(seq, val)
     // early exit for the key: '' nodes we write to reset the db
-    if (!node.value && !node.key) return cb(null, null)
+    if (node.value === null && node.key === '') return cb(null, null)
     cb(null, node)
   }
 }
diff --git a/lib/history.js b/lib/history.js
index 840db5e..035f22a 100644
--- a/lib/history.js
+++ b/lib/history.js
@@ -43,6 +43,7 @@ History.prototype._next = function (cb) {
 
   function done (err, node) {
     if (err) return cb(err)
+    if (!node) return cb(null, null)
     cb(null, node.final())
   }
 }

From 1536f02b3e067dedf86c89a5002aca9032e402a8 Mon Sep 17 00:00:00 2001
From: Jun Matsushita <jun@iilab.org>
Date: Mon, 27 Sep 2021 18:27:46 +0200
Subject: [PATCH 2/3] Enable custom separators

---
 index.js        | 12 ++++++++++--
 lib/batch.js    |  3 +--
 lib/del.js      |  4 ++--
 lib/get.js      |  2 +-
 lib/iterator.js |  4 ++--
 lib/node.js     | 18 +++++++++---------
 lib/put.js      |  2 +-
 7 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/index.js b/index.js
index b6acb99..fc6840d 100644
--- a/index.js
+++ b/index.js
@@ -44,6 +44,7 @@ function HyperTrie (storage, key, opts) {
   this.metadata = opts.metadata || null
   this.hash = opts.hash || null
   this.valueEncoding = opts.valueEncoding ? codecs(opts.valueEncoding) : null
+  this.sep = opts.sep || '/'
   this.alwaysUpdate = !!opts.alwaysUpdate
   this.alwaysReconnect = !!opts.alwaysReconnect
   this.subtype = opts.subtype
@@ -249,6 +250,9 @@ HyperTrie.prototype.list = function (prefix, opts, cb) {
 
 HyperTrie.prototype.iterator = function (prefix, opts) {
   if (isOptions(prefix)) return this.iterator('', prefix)
+  opts = Object.assign({}, opts, {
+    sep: this.sep
+  })
   return new Iterator(this, prefix, opts)
 }
 
@@ -277,6 +281,9 @@ HyperTrie.prototype.createDiffStream = function (other, prefix, opts) {
 
 HyperTrie.prototype.get = function (key, opts, cb) {
   if (typeof opts === 'function') return this.get(key, null, opts)
+  opts = Object.assign({}, opts, {
+    sep: this.sep
+  })
   return new Get(this, key, opts, cb)
 }
 
@@ -292,6 +299,7 @@ HyperTrie.prototype.batch = function (ops, cb) {
 HyperTrie.prototype.put = function (key, value, opts, cb) {
   if (typeof opts === 'function') return this.put(key, value, null, opts)
   opts = Object.assign({}, opts, {
+    sep: this.sep,
     batch: null,
     del: 0
   })
@@ -301,6 +309,7 @@ HyperTrie.prototype.put = function (key, value, opts, cb) {
 HyperTrie.prototype.del = function (key, opts, cb) {
   if (typeof opts === 'function') return this.del(key, null, opts)
   opts = Object.assign({}, opts, {
+    sep: this.sep,
     batch: null
   })
   return new Delete(this, key, opts, cb)
@@ -320,14 +329,13 @@ HyperTrie.prototype.getBySeq = function (seq, opts, cb) {
   if (typeof opts === 'function') return this.getBySeq(seq, null, opts)
   if (seq < 1) return process.nextTick(cb, null, null)
   const self = this
-
   const cached = this._cache.get(seq)
   if (cached) return process.nextTick(onnode, null, cached)
   this.feed.get(seq, opts, onnode)
 
   function onnode (err, val) {
     if (err) return cb(err)
-    const node = Node.decode(val, seq, self.valueEncoding, self.hash)
+    const node = Node.decode(val, seq, self.valueEncoding, self.hash, self.sep)
     self._cache.set(seq, val)
     // early exit for the key: '' nodes we write to reset the db
     if (node.value === null && node.key === '') return cb(null, null)
diff --git a/lib/batch.js b/lib/batch.js
index 0e368a0..2a5f3bc 100644
--- a/lib/batch.js
+++ b/lib/batch.js
@@ -63,7 +63,6 @@ Batch.prototype._start = function () {
 Batch.prototype._update = function () {
   var i = 0
   const self = this
-
   loop(null, null)
 
   function loop (err, head) {
@@ -73,6 +72,6 @@ Batch.prototype._update = function () {
 
     const {type, key, value, hidden, flags} = self._ops[i++]
     if (type === 'del') self._op = new Delete(self._db, key, { batch: self, hidden }, loop)
-    else self._op = new Put(self._db, key, value === undefined ? null : value, { batch: self, del: 0, hidden, flags }, loop)
+    else self._op = new Put(self._db, key, value === undefined ? null : value, { batch: self, del: 0, hidden, flags, sep: self._db.sep }, loop)
   }
 }
diff --git a/lib/del.js b/lib/del.js
index 5f9c6ea..c13ae8e 100644
--- a/lib/del.js
+++ b/lib/del.js
@@ -3,7 +3,7 @@ const Node = require('./node')
 
 module.exports = Delete
 
-function Delete (db, key, { batch, condition = null, hidden = false, closest = false }, cb) {
+function Delete (db, key, { batch, condition = null, hidden = false, closest = false, sep }, cb) {
   this._db = db
   this._key = key
   this._callback = cb
@@ -11,7 +11,7 @@ function Delete (db, key, { batch, condition = null, hidden = false, closest = f
   this._put = null
   this._batch = batch
   this._condition = condition
-  this._node = new Node({key, flags: hidden ? Node.Flags.HIDDEN : 0}, null, null, db.hash)
+  this._node = new Node({key, flags: hidden ? Node.Flags.HIDDEN : 0}, null, null, db.hash, sep)
   this._length = this._node.length
   this._returnClosest = closest
   this._closest = 0
diff --git a/lib/get.js b/lib/get.js
index d75e5e3..4880147 100644
--- a/lib/get.js
+++ b/lib/get.js
@@ -4,7 +4,7 @@ module.exports = Get
 
 function Get (db, key, opts, cb) {
   this._db = db
-  this._node = new Node({key, flags: (opts && opts.hidden) ? Node.Flags.HIDDEN : 0}, 0, null, db.hash)
+  this._node = new Node({key, flags: (opts && opts.hidden) ? Node.Flags.HIDDEN : 0}, 0, null, db.hash, opts.sep)
   this._callback = cb
   this._prefix = !!(opts && opts.prefix)
   this._closest = !!(opts && opts.closest)
diff --git a/lib/iterator.js b/lib/iterator.js
index 81d59d8..51e00a9 100644
--- a/lib/iterator.js
+++ b/lib/iterator.js
@@ -20,7 +20,7 @@ function Iterator (db, prefix, opts) {
   }
 
   this._checkpoint = (opts && opts.checkpoint) || null
-  this._prefix = Node.normalizeKey(prefix || '')
+  this._prefix = Node.normalizeKey(prefix || '', opts.sep)
   this._recursive = !opts || opts.recursive !== false
   this._order = (opts && opts.reverse) ? REVERSE_SORT_ORDER : SORT_ORDER
   this._random = !!(opts && opts.random)
@@ -36,7 +36,7 @@ function Iterator (db, prefix, opts) {
   this._error = null
   this._gt = !!(opts && opts.gt)
   this._needsSort = []
-  this._options = opts ? { extension: opts.extension, wait: opts.wait, timeout: opts.timeout, hidden: !!opts.hidden, onseq: opts.onseq, onwait: null } : { onwait: null }
+  this._options = opts ? { extension: opts.extension, wait: opts.wait, timeout: opts.timeout, hidden: !!opts.hidden, onseq: opts.onseq, onwait: null, sep: opts.sep } : { onwait: null, sep: opts.sep }
   this._flags = (this._recursive ? 1 : 0) | (this._order === REVERSE_SORT_ORDER ? 2 : 0) | (this._gt ? 4 : 0) | ((this._options && this._options.hidden) ? 8 : 0)
   if (this._extensionState) this._options.onwait = this._sendExt.bind(this)
 }
diff --git a/lib/node.js b/lib/node.js
index 49f8eaf..b2e5e05 100644
--- a/lib/node.js
+++ b/lib/node.js
@@ -11,11 +11,11 @@ const Flags = {
   HIDDEN: 1
 }
 
-function Node (data, seq, enc, userHash) {
+function Node (data, seq, enc, userHash, sep) {
   this.seq = seq || 0
-  this.key = normalizeKey(data.key)
+  this.key = normalizeKey(data.key, sep)
   this.value = data.value !== undefined ? data.value : null
-  this.keySplit = split(this.key)
+  this.keySplit = split(this.key, sep)
   this.hash = userHash ? userHash(this.key) : hash(this.keySplit)
   this.trie = data.trieBuffer ? trie.decode(data.trieBuffer) : (data.trie || [])
   this.trieBuffer = null
@@ -88,8 +88,8 @@ Node.prototype.collides = function (node, i) {
   return this.keySplit[j] !== node.keySplit[j]
 }
 
-Node.decode = function (buf, seq, enc, hash) {
-  return new Node(messages.Node.decode(buf), seq, enc, hash)
+Node.decode = function (buf, seq, enc, hash, sep) {
+  return new Node(messages.Node.decode(buf), seq, enc, hash, sep)
 }
 
 Node.terminator = function (i) {
@@ -110,16 +110,16 @@ function hash (keys) {
   return buf
 }
 
-function split (key) {
-  const list = key.split('/')
+function split (key, sep) {
+  const list = key.split(sep)
   if (list[0] === '') list.shift()
   if (list[list.length - 1] === '') list.pop()
   return list
 }
 
-function normalizeKey (key) {
+function normalizeKey (key, sep) {
   if (!key.length) return ''
-  return key[0] === '/' ? key.slice(1) : key
+  return key[0] === sep ? key.slice(1) : key
 }
 
 function defaultStylize (val) {
diff --git a/lib/put.js b/lib/put.js
index 701762a..0c2eb42 100644
--- a/lib/put.js
+++ b/lib/put.js
@@ -20,7 +20,7 @@ function Put (db, key, value, opts, cb) {
   // The flags are shifted in order to both hide the internal flags and support user-defined flags.
   flags = (flags << 8) | (hidden ? Node.Flags.HIDDEN : 0)
 
-  this._node = new Node({key, value, valueBuffer, flags}, 0, db.valueEncoding, db.hash)
+  this._node = new Node({key, value, valueBuffer, flags}, 0, db.valueEncoding, db.hash, opts.sep)
   this._callback = cb
   this._release = null
   this._batch = batch

From eba587d723bf94a91d26c81a17234740d44441bd Mon Sep 17 00:00:00 2001
From: Jun Matsushita <jun@iilab.org>
Date: Mon, 27 Sep 2021 18:28:29 +0200
Subject: [PATCH 3/3] Test custom separator with iterator suite

---
 test/separator.js | 171 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 171 insertions(+)
 create mode 100644 test/separator.js

diff --git a/test/separator.js b/test/separator.js
new file mode 100644
index 0000000..56eb9b6
--- /dev/null
+++ b/test/separator.js
@@ -0,0 +1,171 @@
+const tape = require('tape')
+const create = require('./helpers/create')
+
+const sep = Buffer.alloc(1)
+
+tape('basic iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = ['a', 'b', 'c']
+  const expected = toMap(vals)
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator(), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('iterate a big db', function (t) {
+  const db = create(null, {sep})
+  const vals = range(1000, '#')
+  const expected = toMap(vals)
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator(), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('prefix basic iteration', function (t) {
+  const db = create(null, {sep})
+  var vals = ['foo' + sep + 'a', 'foo' + sep + 'b', 'foo' + sep + 'c']
+  const expected = toMap(vals)
+
+  vals = vals.concat(['a', 'b', 'c'])
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator('foo'), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('empty prefix iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = ['foo/a', 'foo/b', 'foo/c']
+  const expected = {}
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator('bar'), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('prefix iterate a big db', function (t) {
+  var vals = range(1000, 'foo' + sep + '#')
+  const db = create(null, {sep})
+  const expected = toMap(vals)
+
+  vals = vals.concat(range(1000, '#'))
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator('foo'), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('non recursive iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = [
+    'a',
+    'a' + sep + 'b' + sep + 'c' + sep + 'd',
+    'a' + sep + 'b',
+    'b',
+    'b' + sep + 'b' + sep + 'c',
+    'c' + sep + 'a',
+    'c'
+  ]
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator({recursive: false}), function (err, map) {
+      t.error(err, 'no error')
+      // console.log('map', map)
+      const keys = Object.keys(map).map(k => k.split(sep)[0])
+      t.same(keys.sort(), ['a', 'b', 'c'], 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('mixed nested and non nexted iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = ['a', 'a' + sep + 'a', 'a' + sep + 'b', 'a' + sep + 'c', 'a' + sep + 'a' + sep + 'a', 'a' + sep + 'a' + sep + 'b', 'a' + sep + 'a' + sep + 'c']
+  const expected = toMap(vals)
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator(), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('list buffers an iterator', function (t) {
+  const db = create(null, {sep})
+
+  put(db, ['a', 'b', 'b' + sep + 'c'], function (err) {
+    t.error(err, 'no error')
+    db.list(function (err, all) {
+      t.error(err, 'no error')
+      t.same(all.map(v => v.key).sort(), ['a', 'b', 'b' + sep + 'c'])
+      db.list('b', {gt: true}, function (err, all) {
+        t.error(err, 'no error')
+        t.same(all.length, 1)
+        t.same(all[0].key, 'b' + sep + 'c')
+        t.end()
+      })
+    })
+  })
+})
+
+function range (n, v) {
+  // #0, #1, #2, ...
+  return new Array(n).join('.').split('.').map((a, i) => v + i)
+}
+
+function toMap (list) {
+  const map = {}
+  for (var i = 0; i < list.length; i++) {
+    map[list[i]] = list[i]
+  }
+  return map
+}
+
+function all (ite, cb) {
+  const vals = {}
+
+  ite.next(function loop (err, node) {
+    if (err) return cb(err)
+    if (!node) return cb(null, vals)
+    const key = Array.isArray(node) ? node[0].key : node.key
+    // console.log('node', node)
+    if (vals[key]) return cb(new Error('duplicate node for ' + key))
+    vals[key] = Array.isArray(node) ? node.map(n => n.value).sort() : node.value
+    ite.next(loop)
+  })
+}
+
+function put (db, vals, cb) {
+  db.batch(vals.map(v => ({key: v, value: v})), cb)
+}