Skip to content

Commit 9d70527

Browse files
rkapkarkapka
authored
Restore database CLI command (prysmaticlabs#8061)
* restore beacon node db * revert image name * move restore out of the kv folder * remove files from kv folder * go mod tidy * Remove usage of prometheus testutil * add yes/no to prompt text * restore slasher db * organize imports * go mod tidy * restore validator db * close slasher db * defer close backup db in tests * simplify function literal
1 parent fbbdd94 commit 9d70527

27 files changed

+541
-19
lines changed

‎beacon-chain/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ go_library(
1414
importpath = "github.com/prysmaticlabs/prysm/beacon-chain",
1515
visibility = ["//beacon-chain:__subpackages__"],
1616
deps = [
17+
"//beacon-chain/db:go_default_library",
1718
"//beacon-chain/flags:go_default_library",
1819
"//beacon-chain/node:go_default_library",
1920
"//shared/cmd:go_default_library",

‎beacon-chain/db/BUILD.bazel

+8
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ go_library(
1111
name = "go_default_library",
1212
srcs = [
1313
"alias.go",
14+
"cmd.go",
15+
"restore.go",
1416
] + select({
1517
":kafka_disabled": [
1618
"db.go",
@@ -29,7 +31,13 @@ go_library(
2931
"//beacon-chain/cache:go_default_library",
3032
"//beacon-chain/db/iface:go_default_library",
3133
"//beacon-chain/db/kv:go_default_library",
34+
"//shared/cmd:go_default_library",
35+
"//shared/fileutil:go_default_library",
36+
"//shared/promptutil:go_default_library",
37+
"//shared/tos:go_default_library",
38+
"@com_github_pkg_errors//:go_default_library",
3239
"@com_github_sirupsen_logrus//:go_default_library",
40+
"@com_github_urfave_cli_v2//:go_default_library",
3341
] + select({
3442
"//conditions:default": [
3543
"//beacon-chain/db/kafka:go_default_library",

‎beacon-chain/db/cmd.go

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package db
2+
3+
import (
4+
"github.com/prysmaticlabs/prysm/shared/cmd"
5+
"github.com/prysmaticlabs/prysm/shared/tos"
6+
"github.com/sirupsen/logrus"
7+
"github.com/urfave/cli/v2"
8+
)
9+
10+
// DatabaseCommands for Prysm beacon node.
11+
var DatabaseCommands = &cli.Command{
12+
Name: "db",
13+
Category: "db",
14+
Usage: "defines commands for interacting with eth2 beacon node database",
15+
Subcommands: []*cli.Command{
16+
{
17+
Name: "restore",
18+
Description: `restores a database from a backup file`,
19+
Flags: cmd.WrapFlags([]cli.Flag{
20+
cmd.RestoreSourceFileFlag,
21+
cmd.RestoreTargetDirFlag,
22+
}),
23+
Before: tos.VerifyTosAcceptedOrPrompt,
24+
Action: func(cliCtx *cli.Context) error {
25+
if err := restore(cliCtx); err != nil {
26+
logrus.Fatalf("Could not restore database: %v", err)
27+
}
28+
return nil
29+
},
30+
},
31+
},
32+
}

‎beacon-chain/db/kv/backup_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func TestStore_Backup(t *testing.T) {
3434
require.NoError(t, db.Close(), "Failed to close database")
3535

3636
oldFilePath := filepath.Join(backupsPath, files[0].Name())
37-
newFilePath := filepath.Join(backupsPath, databaseFileName)
37+
newFilePath := filepath.Join(backupsPath, DatabaseFileName)
3838
// We rename the file to match the database file name
3939
// our NewKVStore function expects when opening a database.
4040
require.NoError(t, os.Rename(oldFilePath, newFilePath))

‎beacon-chain/db/kv/kv.go

+9-5
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,13 @@ const (
2424
// VotesCacheSize with 1M validators will be 8MB.
2525
VotesCacheSize = 1 << 23
2626
// NumOfVotes specifies the vote cache size.
27-
NumOfVotes = 1 << 20
28-
databaseFileName = "beaconchain.db"
29-
boltAllocSize = 8 * 1024 * 1024
27+
NumOfVotes = 1 << 20
28+
// BeaconNodeDbDirName is the name of the directory containing the beacon node database.
29+
BeaconNodeDbDirName = "beaconchaindata"
30+
// DatabaseFileName is the name of the beacon node database.
31+
DatabaseFileName = "beaconchain.db"
32+
33+
boltAllocSize = 8 * 1024 * 1024
3034
)
3135

3236
// BlockCacheSize specifies 1000 slots worth of blocks cached, which
@@ -56,7 +60,7 @@ func NewKVStore(dirPath string, stateSummaryCache *cache.StateSummaryCache) (*St
5660
return nil, err
5761
}
5862
}
59-
datafile := path.Join(dirPath, databaseFileName)
63+
datafile := path.Join(dirPath, DatabaseFileName)
6064
boltDB, err := bolt.Open(datafile, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{Timeout: 1 * time.Second, InitialMmapSize: 10e6})
6165
if err != nil {
6266
if errors.Is(err, bolt.ErrTimeout) {
@@ -134,7 +138,7 @@ func (s *Store) ClearDB() error {
134138
return nil
135139
}
136140
prometheus.Unregister(createBoltCollector(s.db))
137-
if err := os.Remove(path.Join(s.databasePath, databaseFileName)); err != nil {
141+
if err := os.Remove(path.Join(s.databasePath, DatabaseFileName)); err != nil {
138142
return errors.Wrap(err, "could not remove database file")
139143
}
140144
return nil

‎beacon-chain/db/restore.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package db
2+
3+
import (
4+
"os"
5+
"path"
6+
"strings"
7+
8+
"github.com/pkg/errors"
9+
"github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
10+
"github.com/prysmaticlabs/prysm/shared/cmd"
11+
"github.com/prysmaticlabs/prysm/shared/fileutil"
12+
"github.com/prysmaticlabs/prysm/shared/promptutil"
13+
"github.com/sirupsen/logrus"
14+
"github.com/urfave/cli/v2"
15+
)
16+
17+
const dbExistsYesNoPrompt = "A database file already exists in the target directory. " +
18+
"Are you sure that you want to overwrite it? [y/n]"
19+
20+
func restore(cliCtx *cli.Context) error {
21+
sourceFile := cliCtx.String(cmd.RestoreSourceFileFlag.Name)
22+
targetDir := cliCtx.String(cmd.RestoreTargetDirFlag.Name)
23+
24+
restoreDir := path.Join(targetDir, kv.BeaconNodeDbDirName)
25+
if fileutil.FileExists(path.Join(restoreDir, kv.DatabaseFileName)) {
26+
resp, err := promptutil.ValidatePrompt(
27+
os.Stdin, dbExistsYesNoPrompt, promptutil.ValidateYesOrNo,
28+
)
29+
if err != nil {
30+
return errors.Wrap(err, "could not validate choice")
31+
}
32+
if strings.ToLower(resp) == "n" {
33+
logrus.Info("Restore aborted")
34+
return nil
35+
}
36+
}
37+
if err := fileutil.MkdirAll(restoreDir); err != nil {
38+
return err
39+
}
40+
if err := fileutil.CopyFile(sourceFile, path.Join(restoreDir, kv.DatabaseFileName)); err != nil {
41+
return err
42+
}
43+
44+
logrus.Info("Restore completed successfully")
45+
return nil
46+
}

‎beacon-chain/db/restore_test.go

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package db
2+
3+
import (
4+
"context"
5+
"flag"
6+
"io/ioutil"
7+
"os"
8+
"path"
9+
"testing"
10+
11+
"github.com/prysmaticlabs/prysm/beacon-chain/cache"
12+
"github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
13+
"github.com/prysmaticlabs/prysm/shared/cmd"
14+
"github.com/prysmaticlabs/prysm/shared/testutil"
15+
"github.com/prysmaticlabs/prysm/shared/testutil/assert"
16+
"github.com/prysmaticlabs/prysm/shared/testutil/require"
17+
logTest "github.com/sirupsen/logrus/hooks/test"
18+
"github.com/urfave/cli/v2"
19+
)
20+
21+
func TestRestore(t *testing.T) {
22+
logHook := logTest.NewGlobal()
23+
ctx := context.Background()
24+
25+
backupDb, err := kv.NewKVStore(t.TempDir(), cache.NewStateSummaryCache())
26+
defer func() {
27+
require.NoError(t, backupDb.Close())
28+
}()
29+
require.NoError(t, err)
30+
head := testutil.NewBeaconBlock()
31+
head.Block.Slot = 5000
32+
require.NoError(t, backupDb.SaveBlock(ctx, head))
33+
root, err := head.Block.HashTreeRoot()
34+
require.NoError(t, err)
35+
st := testutil.NewBeaconState()
36+
require.NoError(t, backupDb.SaveState(ctx, st, root))
37+
require.NoError(t, backupDb.SaveHeadBlockRoot(ctx, root))
38+
require.NoError(t, err)
39+
require.NoError(t, backupDb.Close())
40+
// We rename the backup file so that we can later verify
41+
// whether the restored db has been renamed correctly.
42+
require.NoError(t, os.Rename(
43+
path.Join(backupDb.DatabasePath(), kv.DatabaseFileName),
44+
path.Join(backupDb.DatabasePath(), "backup.db")))
45+
46+
restoreDir := t.TempDir()
47+
app := cli.App{}
48+
set := flag.NewFlagSet("test", 0)
49+
set.String(cmd.RestoreSourceFileFlag.Name, "", "")
50+
set.String(cmd.RestoreTargetDirFlag.Name, "", "")
51+
require.NoError(t, set.Set(cmd.RestoreSourceFileFlag.Name, path.Join(backupDb.DatabasePath(), "backup.db")))
52+
require.NoError(t, set.Set(cmd.RestoreTargetDirFlag.Name, restoreDir))
53+
cliCtx := cli.NewContext(&app, set, nil)
54+
55+
assert.NoError(t, restore(cliCtx))
56+
57+
files, err := ioutil.ReadDir(path.Join(restoreDir, kv.BeaconNodeDbDirName))
58+
require.NoError(t, err)
59+
assert.Equal(t, 1, len(files))
60+
assert.Equal(t, kv.DatabaseFileName, files[0].Name())
61+
restoredDb, err := kv.NewKVStore(path.Join(restoreDir, kv.BeaconNodeDbDirName), nil)
62+
defer func() {
63+
require.NoError(t, restoredDb.Close())
64+
}()
65+
require.NoError(t, err)
66+
headBlock, err := restoredDb.HeadBlock(ctx)
67+
require.NoError(t, err)
68+
assert.Equal(t, uint64(5000), headBlock.Block.Slot, "Restored database has incorrect data")
69+
assert.LogsContain(t, logHook, "Restore completed successfully")
70+
71+
}

‎beacon-chain/main.go

+6
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
gethlog "github.com/ethereum/go-ethereum/log"
1111
golog "github.com/ipfs/go-log/v2"
1212
joonix "github.com/joonix/log"
13+
"github.com/prysmaticlabs/prysm/beacon-chain/db"
1314
"github.com/prysmaticlabs/prysm/beacon-chain/flags"
1415
"github.com/prysmaticlabs/prysm/beacon-chain/node"
1516
"github.com/prysmaticlabs/prysm/shared/cmd"
@@ -100,6 +101,8 @@ var appFlags = []cli.Flag{
100101
cmd.ChainConfigFileFlag,
101102
cmd.GrpcMaxCallRecvMsgSizeFlag,
102103
cmd.AcceptTosFlag,
104+
cmd.RestoreSourceFileFlag,
105+
cmd.RestoreTargetDirFlag,
103106
}
104107

105108
func init() {
@@ -113,6 +116,9 @@ func main() {
113116
app.Usage = "this is a beacon chain implementation for Ethereum 2.0"
114117
app.Action = startNode
115118
app.Version = version.GetVersion()
119+
app.Commands = []*cli.Command{
120+
db.DatabaseCommands,
121+
}
116122

117123
app.Flags = appFlags
118124

‎beacon-chain/node/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ go_library(
1414
"//beacon-chain/cache:go_default_library",
1515
"//beacon-chain/cache/depositcache:go_default_library",
1616
"//beacon-chain/db:go_default_library",
17+
"//beacon-chain/db/kv:go_default_library",
1718
"//beacon-chain/flags:go_default_library",
1819
"//beacon-chain/forkchoice:go_default_library",
1920
"//beacon-chain/forkchoice/protoarray:go_default_library",

‎beacon-chain/node/node.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/prysmaticlabs/prysm/beacon-chain/cache"
2222
"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
2323
"github.com/prysmaticlabs/prysm/beacon-chain/db"
24+
"github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
2425
"github.com/prysmaticlabs/prysm/beacon-chain/flags"
2526
"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice"
2627
"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
@@ -54,7 +55,6 @@ import (
5455

5556
var log = logrus.WithField("prefix", "node")
5657

57-
const beaconChainDBName = "beaconchaindata"
5858
const testSkipPowFlag = "test-skip-pow"
5959

6060
// BeaconNode defines a struct that handles the services running a random beacon chain
@@ -291,7 +291,7 @@ func (b *BeaconNode) startForkChoice() {
291291

292292
func (b *BeaconNode) startDB(cliCtx *cli.Context) error {
293293
baseDir := cliCtx.String(cmd.DataDirFlag.Name)
294-
dbPath := filepath.Join(baseDir, beaconChainDBName)
294+
dbPath := filepath.Join(baseDir, kv.BeaconNodeDbDirName)
295295
clearDB := cliCtx.Bool(cmd.ClearDB.Name)
296296
forceClearDB := cliCtx.Bool(cmd.ForceClearDB.Name)
297297

‎beacon-chain/usage.go

+2
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ var appHelpFlagGroups = []flagGroup{
7070
cmd.ChainConfigFileFlag,
7171
cmd.GrpcMaxCallRecvMsgSizeFlag,
7272
cmd.AcceptTosFlag,
73+
cmd.RestoreSourceFileFlag,
74+
cmd.RestoreTargetDirFlag,
7375
},
7476
},
7577
{

‎shared/cmd/flags.go

+12
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,18 @@ var (
219219
Name: "accept-terms-of-use",
220220
Usage: "Accept Terms and Conditions (for non-interactive environments)",
221221
}
222+
// RestoreSourceFileFlag specifies the filepath to the backed-up database file
223+
// which will be used to restore the database.
224+
RestoreSourceFileFlag = &cli.StringFlag{
225+
Name: "restore-source-file",
226+
Usage: "Filepath to the backed-up database file which will be used to restore the database",
227+
}
228+
// RestoreTargetDirFlag specifies the target directory of the restored database.
229+
RestoreTargetDirFlag = &cli.StringFlag{
230+
Name: "restore-target-dir",
231+
Usage: "Target directory of the restored database",
232+
Value: DefaultDataDir(),
233+
}
222234
)
223235

224236
// LoadFlagsFromConfig sets flags values from config file if ConfigFileFlag is set.

‎slasher/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ go_library(
2020
"//shared/logutil:go_default_library",
2121
"//shared/tos:go_default_library",
2222
"//shared/version:go_default_library",
23+
"//slasher/db:go_default_library",
2324
"//slasher/flags:go_default_library",
2425
"//slasher/node:go_default_library",
2526
"@com_github_joonix_log//:go_default_library",

‎slasher/db/BUILD.bazel

+21-2
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,38 @@ go_library(
55
name = "go_default_library",
66
srcs = [
77
"alias.go",
8+
"cmd.go",
89
"db.go",
10+
"restore.go",
911
],
1012
importpath = "github.com/prysmaticlabs/prysm/slasher/db",
1113
visibility = ["//slasher:__subpackages__"],
1214
deps = [
15+
"//shared/cmd:go_default_library",
16+
"//shared/fileutil:go_default_library",
17+
"//shared/promptutil:go_default_library",
18+
"//shared/tos:go_default_library",
1319
"//slasher/db/iface:go_default_library",
1420
"//slasher/db/kv:go_default_library",
21+
"@com_github_pkg_errors//:go_default_library",
22+
"@com_github_sirupsen_logrus//:go_default_library",
23+
"@com_github_urfave_cli_v2//:go_default_library",
1524
],
1625
)
1726

1827
go_test(
1928
name = "go_default_test",
20-
srcs = ["db_test.go"],
29+
srcs = [
30+
"db_test.go",
31+
"restore_test.go",
32+
],
2133
embed = [":go_default_library"],
22-
deps = ["//slasher/db/kv:go_default_library"],
34+
deps = [
35+
"//shared/cmd:go_default_library",
36+
"//shared/testutil/assert:go_default_library",
37+
"//shared/testutil/require:go_default_library",
38+
"//slasher/db/kv:go_default_library",
39+
"@com_github_sirupsen_logrus//hooks/test:go_default_library",
40+
"@com_github_urfave_cli_v2//:go_default_library",
41+
],
2342
)

‎slasher/db/cmd.go

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package db
2+
3+
import (
4+
"github.com/prysmaticlabs/prysm/shared/cmd"
5+
"github.com/prysmaticlabs/prysm/shared/tos"
6+
"github.com/sirupsen/logrus"
7+
"github.com/urfave/cli/v2"
8+
)
9+
10+
// DatabaseCommands for Prysm slasher.
11+
var DatabaseCommands = &cli.Command{
12+
Name: "db",
13+
Category: "db",
14+
Usage: "defines commands for interacting with eth2 slasher database",
15+
Subcommands: []*cli.Command{
16+
{
17+
Name: "restore",
18+
Description: `restores a database from a backup file`,
19+
Flags: cmd.WrapFlags([]cli.Flag{
20+
cmd.RestoreSourceFileFlag,
21+
cmd.RestoreTargetDirFlag,
22+
}),
23+
Before: tos.VerifyTosAcceptedOrPrompt,
24+
Action: func(cliCtx *cli.Context) error {
25+
if err := restore(cliCtx); err != nil {
26+
logrus.Fatalf("Could not restore database: %v", err)
27+
}
28+
return nil
29+
},
30+
},
31+
},
32+
}

0 commit comments

Comments
 (0)