Skip to content

Commit b2f1b72

Browse files
bbengfortbbengfort
authored
Storage Engine (#40)
1 parent 7e3f9ee commit b2f1b72

32 files changed

+1461
-25
lines changed

‎go.mod

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,25 @@
11
module github.com/rotationalio/honu
22

3-
go 1.23.1
3+
go 1.23.3
44

55
require (
66
github.com/cenkalti/backoff v2.2.1+incompatible
77
github.com/google/go-querystring v1.1.0
88
github.com/joho/godotenv v1.5.1
99
github.com/julienschmidt/httprouter v1.3.0
10-
github.com/oklog/ulid/v2 v2.1.0
1110
github.com/rotationalio/confire v1.1.0
1211
github.com/rs/zerolog v1.33.0
1312
github.com/stretchr/testify v1.9.0
13+
github.com/syndtr/goleveldb v1.0.0
1414
github.com/tinylib/msgp v1.2.4
1515
github.com/urfave/cli/v2 v2.27.5
16+
go.rtnl.ai/ulid v1.1.1
1617
)
1718

1819
require (
1920
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
2021
github.com/davecgh/go-spew v1.1.1 // indirect
22+
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
2123
github.com/mattn/go-colorable v0.1.13 // indirect
2224
github.com/mattn/go-isatty v0.0.19 // indirect
2325
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect

‎go.sum

+27-3
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,17 @@ github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB
55
github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
66
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
77
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
8+
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
89
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
10+
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
11+
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
12+
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
913
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
1014
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
1115
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
1216
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
17+
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
18+
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
1319
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
1420
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
1521
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
@@ -19,9 +25,11 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
1925
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
2026
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
2127
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
22-
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
23-
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
24-
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
28+
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
29+
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
30+
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
31+
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
32+
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
2533
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY=
2634
github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
2735
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -36,18 +44,34 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
3644
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
3745
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
3846
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
47+
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
48+
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
3949
github.com/tinylib/msgp v1.2.4 h1:yLFeUGostXXSGW5vxfT5dXG/qzkn4schv2I7at5+hVU=
4050
github.com/tinylib/msgp v1.2.4/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
4151
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
4252
github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
4353
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
4454
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
55+
go.rtnl.ai/ulid v1.1.1 h1:7zDInpWEeiKFcDoJyonails5lQxwOr4wc8K4TAuOVNE=
56+
go.rtnl.ai/ulid v1.1.1/go.mod h1:F95yPYwEEZdz5sM4GC6buziff6xD+7XVcGZ/8n37Cn0=
57+
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
58+
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
59+
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
60+
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
4561
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
4662
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
4763
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
4864
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
65+
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
66+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
4967
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
5068
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
5169
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
70+
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
71+
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
72+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
73+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
74+
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
75+
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
5276
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
5377
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎pkg/api/v1/client.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ import (
1414

1515
"github.com/cenkalti/backoff"
1616
"github.com/google/go-querystring/query"
17-
"github.com/oklog/ulid/v2"
1817
"github.com/rotationalio/honu/pkg/api/v1/credentials"
1918
"github.com/rs/zerolog/log"
19+
"go.rtnl.ai/ulid"
2020
)
2121

2222
const (

‎pkg/config/config.go

+7
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,23 @@ const Prefix = "honu"
1818
// values that are omitted. The Config should be validated in preparation for running
1919
// the honudb instance to ensure that all server operations work as expected.
2020
type Config struct {
21+
PID uint32 `required:"true" desc:"the unique process id for this replica"`
2122
Maintenance bool `default:"false" desc:"if true, the replica will start in maintenance mode"`
2223
LogLevel logger.LevelDecoder `split_words:"true" default:"info" desc:"specify the verbosity of logging (trace, debug, info, warn, error, fatal panic)"`
2324
ConsoleLog bool `split_words:"true" default:"false" desc:"if true logs colorized human readable output instead of json"`
2425
BindAddr string `split_words:"true" default:":3264" desc:"the ip address and port to bind the honu database server on"`
2526
ReadTimeout time.Duration `split_words:"true" default:"20s" desc:"amount of time allowed to read request headers before server decides the request is too slow"`
2627
WriteTimeout time.Duration `split_words:"true" default:"20s" desc:"maximum amount of time before timing out a write to a response"`
2728
IdleTimeout time.Duration `split_words:"true" default:"10m" desc:"maximum amount of time to wait for the next request while keep alives are enabled"`
29+
Store StoreConfig
2830
processed bool
2931
}
3032

33+
type StoreConfig struct {
34+
ReadOnly bool `default:"false" split_words:"false" desc:"open the the underlying data store in read-only mode"`
35+
DataPath string `required:"true" split_words:"true" desc:"path to directory where data is stored (created if it doesn't exist)"`
36+
}
37+
3138
func New() (conf Config, err error) {
3239
if err = confire.Process(Prefix, &conf); err != nil {
3340
return Config{}, err

‎pkg/config/config_test.go

+10-4
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@ import (
1010
)
1111

1212
var testEnv = map[string]string{
13-
"HONU_MAINTENANCE": "true",
14-
"HONU_LOG_LEVEL": "debug",
15-
"HONU_CONSOLE_LOG": "true",
16-
"HONU_BIND_ADDR": "127.0.0.1:443",
13+
"HONU_PID": "24",
14+
"HONU_MAINTENANCE": "true",
15+
"HONU_LOG_LEVEL": "debug",
16+
"HONU_CONSOLE_LOG": "true",
17+
"HONU_BIND_ADDR": "127.0.0.1:443",
18+
"HONU_STORE_READONLY": "true",
19+
"HONU_STORE_DATA_PATH": "/tmp/honu",
1720
}
1821

1922
func TestConfig(t *testing.T) {
@@ -26,10 +29,13 @@ func TestConfig(t *testing.T) {
2629
require.False(t, conf.IsZero(), "processed config should not be zero valued")
2730

2831
// Ensure configuration is correctly set from the environment
32+
require.Equal(t, uint32(24), conf.PID)
2933
require.True(t, conf.Maintenance)
3034
require.Equal(t, zerolog.DebugLevel, conf.GetLogLevel())
3135
require.True(t, conf.ConsoleLog)
3236
require.Equal(t, testEnv["HONU_BIND_ADDR"], conf.BindAddr)
37+
require.True(t, conf.Store.ReadOnly)
38+
require.Equal(t, testEnv["HONU_STORE_DATA_PATH"], conf.Store.DataPath)
3339
}
3440

3541
// Returns the current environment for the specified keys, or if no keys are specified

‎pkg/store/engine/engine.go

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package engine
2+
3+
import (
4+
"github.com/rotationalio/honu/pkg/store/iterator"
5+
"github.com/rotationalio/honu/pkg/store/key"
6+
"github.com/rotationalio/honu/pkg/store/object"
7+
)
8+
9+
// Engines are the disk storage mechanism that Honu wraps. Users may chose different
10+
// engines for a variety of reasons, including variable performance benefits, different
11+
// features, or even implement heterogeneous Honu networks composed of different engines.
12+
type Engine interface {
13+
// Engine returns the engine name and is used for debugging and logging.
14+
Engine() string
15+
16+
// Close the engine so that it no longer can be accessed.
17+
Close() error
18+
}
19+
20+
// Store is a simple key/value interface that allows for Get, Put, and Delete. Nearly
21+
// all engines should support the Store interface. Note that transactions and options
22+
// are a higher level construct than engine stores and are provided by the Honu store.
23+
type Store interface {
24+
Has(key.Key) (exists bool, err error)
25+
Get(key.Key) (object.Object, error)
26+
Put(key.Key, object.Object) error
27+
Delete(key.Key) error
28+
}
29+
30+
// Iterator engines allow queries that scan a range of consecutive keys.
31+
type Iterator interface {
32+
Iter(prefix []byte) (i iterator.Iterator, err error)
33+
Range(start, limit []byte) (i iterator.Iterator, err error)
34+
}

‎pkg/store/engine/errors.go

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package engine
2+
3+
import "errors"
4+
5+
var (
6+
ErrNotFound = errors.New("object not found")
7+
ErrReadOnlyDB = errors.New("cannot execute operation in readonly mode")
8+
ErrReadOnlyTx = errors.New("cannot execute operation: transaction is read only")
9+
ErrClosed = errors.New("database engine has been closed")
10+
ErrAlreadyExists = errors.New("specified key already exists")
11+
ErrNotSupported = errors.New("operation not supported")
12+
)

‎pkg/store/engine/leveldb/iter.go

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package leveldb
2+
3+
import (
4+
"github.com/rotationalio/honu/pkg/store/iterator"
5+
"github.com/rotationalio/honu/pkg/store/key"
6+
"github.com/rotationalio/honu/pkg/store/object"
7+
8+
ldbiter "github.com/syndtr/goleveldb/leveldb/iterator"
9+
)
10+
11+
func NewIterator(iter ldbiter.Iterator) iterator.Iterator {
12+
return &ldbIterator{Iterator: iter}
13+
}
14+
15+
type ldbIterator struct {
16+
ldbiter.Iterator
17+
}
18+
19+
// Iterator accessor methods.
20+
func (i *ldbIterator) Key() key.Key { return key.Key(i.Iterator.Key()) }
21+
func (i *ldbIterator) Object() object.Object { return object.Object(i.Iterator.Value()) }
22+
func (i *ldbIterator) Error() error { return Wrap(i.Iterator.Error()) }

‎pkg/store/engine/leveldb/iter_test.go

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package leveldb_test
2+
3+
import (
4+
"math/rand/v2"
5+
"testing"
6+
7+
"github.com/rotationalio/honu/pkg/store/iterator"
8+
"github.com/rotationalio/honu/pkg/store/key"
9+
"github.com/stretchr/testify/require"
10+
"go.rtnl.ai/ulid"
11+
)
12+
13+
func TestIterator(t *testing.T) {
14+
// Create a database and populate it with some data
15+
db := OpenLevelDB(t, false)
16+
17+
// Testing Data Structures
18+
cid := ulid.Make()
19+
oids := make(map[ulid.ULID]int)
20+
nkeys := 0
21+
22+
// Create three different objects with multiple versions each in the same collection
23+
var target ulid.ULID
24+
for i := 0; i < 3; i++ {
25+
count := rand.IntN(20) + 4
26+
nkeys += count
27+
28+
oid := ulid.Make()
29+
oids[oid] = count
30+
31+
Populate(t, db, count, cid, oid)
32+
33+
if i == 1 {
34+
target = oid
35+
}
36+
}
37+
38+
t.Run("All", func(t *testing.T) {
39+
actual := 0
40+
iter, err := db.Iter(nil)
41+
require.NoError(t, err)
42+
43+
for iter.Next() {
44+
actual++
45+
46+
k := iter.Key()
47+
require.Equal(t, cid, k.CollectionID())
48+
49+
obj := iter.Object()
50+
require.NotNil(t, obj)
51+
}
52+
53+
iter.Release()
54+
require.NoError(t, iter.Error())
55+
require.Equal(t, nkeys, actual)
56+
})
57+
58+
t.Run("Prefix", func(t *testing.T) {
59+
k := key.New(cid, target, nil)
60+
iter, err := db.Iter(k.ObjectPrefix())
61+
require.NoError(t, err)
62+
63+
actual := 0
64+
for iter.Next() {
65+
actual++
66+
require.Equal(t, target, iter.Key().ObjectID())
67+
}
68+
69+
iter.Release()
70+
require.NoError(t, iter.Error())
71+
require.Equal(t, oids[target], actual)
72+
})
73+
74+
t.Run("Range", func(t *testing.T) {
75+
k := key.New(cid, target, nil)
76+
iter, err := db.Range(k.ObjectPrefix(), k.ObjectLimit())
77+
require.NoError(t, err)
78+
79+
actual := 0
80+
for iter.Next() {
81+
actual++
82+
require.Equal(t, target, iter.Key().ObjectID())
83+
}
84+
85+
iter.Release()
86+
require.NoError(t, iter.Error())
87+
require.Equal(t, oids[target], actual)
88+
})
89+
90+
t.Run("Error", func(t *testing.T) {
91+
iter, err := db.Iter(nil)
92+
require.NoError(t, err)
93+
94+
iter.Release()
95+
require.False(t, iter.Next())
96+
require.ErrorIs(t, iter.Error(), iterator.ErrIterReleased)
97+
})
98+
}

0 commit comments

Comments
 (0)