Skip to content

Commit cebb629

Browse files
prestonvanloonprylabs-bulldozer[bot]
prestonvanloon
and
prylabs-bulldozer[bot]
authored
Add beacon state unmarshal fuzzer, afl support (prysmaticlabs#6625)
* Add AFL third_party libraries * add beacon state fuzzing, add afl fuzz bundle * rm fuzzing engine * fix and lint * Check for array out of bounds when calculating proposer delta * failing test * fix * Checkpoint progress * Add requirement that inclusion distance is not zero, add regression test * No need for HTR since that is covered in process slots * Removing some fuzzit logic, old fuzz tests * Add ssz encoder test and fix * Fuzzing checkpoint, adding fuzzing to the p2p layer * ignore some libfuzzer files * Full testing of p2p processing of blocks, with some mocked stuff * use tmpdir and always process blocks * use checkptr * Update ethereumapis * go mod tidy * benchmarks for ferran's fast ssz hash tree root * Update fastssz * fmt * gaz * goimports * Fix * fix ethereumapis * fix again * kafka * fix gen file * fix compute signing root * gofmt * checkpoint progress * progress * checkpoint * updates * updates * merge fix * WIP * merge * fix build * fix merge related issues * cleanup * revert unrelated * lint * lint * lint * manual tags for fuzz * Commentary on upload script * some import fixes, but not all * fix //fuzz:fuzz_tests * rm unused test * update generated ssz * Set // +build libfuzzer * remove debug code * A bit of refactoring ot explain why there is a committee_disabled file Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>
1 parent e477df3 commit cebb629

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+11751
-873
lines changed

‎.bazelrc

+1-7
Original file line numberDiff line numberDiff line change
@@ -69,18 +69,12 @@ build:fuzz --copt=-fno-omit-frame-pointer
6969
build:fuzz --define=FUZZING_ENGINE=libfuzzer
7070
build:fuzz --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
7171
build:fuzz --linkopt -Wl,--no-as-needed
72-
build:fuzz --define=gc_goopts=-d=libfuzzer
72+
build:fuzz --define=gc_goopts=-d=libfuzzer,checkptr
7373
build:fuzz --run_under=//tools:fuzz_wrapper
7474
build:fuzz --compilation_mode=opt
7575

7676
test:fuzz --local_test_jobs="HOST_CPUS*.5"
7777

78-
test:fuzzit --config=fuzz
79-
test:fuzzit --test_env=FUZZIT_API_KEY
80-
test:fuzzit --test_env=PRYSM_BUILD_IMAGE=gcr.io/prysmaticlabs/prysm-fuzzit:v0.11.0
81-
test:fuzzit --test_timeout=1200
82-
test:fuzzit --run_under=//tools:fuzzit_wrapper
83-
8478
# Build binary with cgo symbolizer for debugging / profiling.
8579
build:cgo_symbolizer --config=llvm
8680
build:cgo_symbolizer --copt=-g

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,7 @@ password.txt
2929

3030
# Dist files
3131
dist
32+
33+
# libfuzzer
34+
oom-*
35+
crash-*

‎beacon-chain/blockchain/BUILD.bazel

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ go_library(
2020
"service.go",
2121
],
2222
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain",
23-
visibility = ["//beacon-chain:__subpackages__"],
23+
visibility = [
24+
"//beacon-chain:__subpackages__",
25+
"//fuzz:__pkg__",
26+
],
2427
deps = [
2528
"//beacon-chain/cache:go_default_library",
2629
"//beacon-chain/cache/depositcache:go_default_library",

‎beacon-chain/cache/BUILD.bazel

+9-2
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,21 @@ go_library(
66
srcs = [
77
"attestation_data.go",
88
"checkpoint_state.go",
9-
"committee.go",
9+
"committees.go",
1010
"common.go",
1111
"doc.go",
1212
"hot_state_cache.go",
1313
"skip_slot_cache.go",
1414
"state_summary.go",
1515
"subnet_ids.go",
16-
],
16+
] + select({
17+
"//fuzz:fuzzing_enabled": [
18+
"committee_disabled.go",
19+
],
20+
"//conditions:default": [
21+
"committee.go",
22+
],
23+
}),
1724
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
1825
visibility = [
1926
"//beacon-chain:__subpackages__",

‎beacon-chain/cache/committee.go

+8-13
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// -build libfuzzer
2+
13
package cache
24

35
import (
@@ -12,10 +14,6 @@ import (
1214
)
1315

1416
var (
15-
// ErrNotCommittee will be returned when a cache object is not a pointer to
16-
// a Committee struct.
17-
ErrNotCommittee = errors.New("object is not a committee struct")
18-
1917
// maxCommitteesCacheSize defines the max number of shuffled committees on per randao basis can cache.
2018
// Due to reorgs and long finality, it's good to keep the old cache around for quickly switch over.
2119
maxCommitteesCacheSize = uint64(32)
@@ -32,15 +30,6 @@ var (
3230
})
3331
)
3432

35-
// Committees defines the shuffled committees seed.
36-
type Committees struct {
37-
CommitteeCount uint64
38-
Seed [32]byte
39-
ShuffledIndices []uint64
40-
SortedIndices []uint64
41-
ProposerIndices []uint64
42-
}
43-
4433
// CommitteeCache is a struct with 1 queue for looking up shuffled indices list by seed.
4534
type CommitteeCache struct {
4635
CommitteeCache *cache.FIFO
@@ -216,6 +205,12 @@ func (c *CommitteeCache) ProposerIndices(seed [32]byte) ([]uint64, error) {
216205
return item.ProposerIndices, nil
217206
}
218207

208+
// HasEntry returns true if the committee cache has a value.
209+
func (c *CommitteeCache) HasEntry(seed string) bool {
210+
_, ok, err := c.CommitteeCache.GetByKey(seed)
211+
return err == nil && ok
212+
}
213+
219214
func startEndIndices(c *Committees, index uint64) (uint64, uint64) {
220215
validatorCount := uint64(len(c.ShuffledIndices))
221216
start := sliceutil.SplitOffset(validatorCount, c.CommitteeCount, index)

‎beacon-chain/cache/committee_disabled.go

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// +build libfuzzer
2+
3+
// This file is used in fuzzer builds to bypass global committee caches.
4+
package cache
5+
6+
// FakeCommitteeCache is a struct with 1 queue for looking up shuffled indices list by seed.
7+
type FakeCommitteeCache struct {
8+
}
9+
10+
// NewCommitteesCache creates a new committee cache for storing/accessing shuffled indices of a committee.
11+
func NewCommitteesCache() *FakeCommitteeCache {
12+
return &FakeCommitteeCache{}
13+
}
14+
15+
// Committee fetches the shuffled indices by slot and committee index. Every list of indices
16+
// represent one committee. Returns true if the list exists with slot and committee index. Otherwise returns false, nil.
17+
func (c *FakeCommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]uint64, error) {
18+
return nil, nil
19+
}
20+
21+
// AddCommitteeShuffledList adds Committee shuffled list object to the cache. T
22+
// his method also trims the least recently list if the cache size has ready the max cache size limit.
23+
func (c *FakeCommitteeCache) AddCommitteeShuffledList(committees *Committees) error {
24+
return nil
25+
}
26+
27+
// AddProposerIndicesList updates the committee shuffled list with proposer indices.
28+
func (c *FakeCommitteeCache) AddProposerIndicesList(seed [32]byte, indices []uint64) error {
29+
return nil
30+
}
31+
32+
// ActiveIndices returns the active indices of a given seed stored in cache.
33+
func (c *FakeCommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
34+
return nil, nil
35+
}
36+
37+
// ActiveIndicesCount returns the active indices count of a given seed stored in cache.
38+
func (c *FakeCommitteeCache) ActiveIndicesCount(seed [32]byte) (int, error) {
39+
return 0, nil
40+
}
41+
42+
// ProposerIndices returns the proposer indices of a given seed.
43+
func (c *FakeCommitteeCache) ProposerIndices(seed [32]byte) ([]uint64, error) {
44+
return nil, nil
45+
}
46+
47+
// HasEntry returns true if the committee cache has a value.
48+
func (c *FakeCommitteeCache) HasEntry(string) bool {
49+
return false
50+
}

‎beacon-chain/cache/committees.go

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package cache
2+
3+
import "errors"
4+
5+
// ErrNotCommittee will be returned when a cache object is not a pointer to
6+
// a Committee struct.
7+
var ErrNotCommittee = errors.New("object is not a committee struct")
8+
9+
// Committees defines the shuffled committees seed.
10+
type Committees struct {
11+
CommitteeCount uint64
12+
Seed [32]byte
13+
ShuffledIndices []uint64
14+
SortedIndices []uint64
15+
ProposerIndices []uint64
16+
}

‎beacon-chain/core/helpers/BUILD.bazel

+3-2
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ go_library(
1818
visibility = [
1919
"//beacon-chain:__subpackages__",
2020
"//endtoend/evaluators:__pkg__",
21+
"//fuzz:__pkg__",
22+
"//shared/attestationutil:__pkg__",
2123
"//shared/benchutil/benchmark_files:__subpackages__",
24+
"//shared/depositutil:__pkg__",
2225
"//shared/interop:__pkg__",
2326
"//shared/keystore:__pkg__",
24-
"//shared/depositutil:__pkg__",
2527
"//shared/p2putils:__pkg__",
26-
"//shared/attestationutil:__pkg__",
2728
"//shared/testutil:__pkg__",
2829
"//slasher:__subpackages__",
2930
"//tools:__subpackages__",

‎beacon-chain/core/helpers/committee.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,8 @@ func UpdateCommitteeCache(state *stateTrie.BeaconState, epoch uint64) error {
312312
if err != nil {
313313
return err
314314
}
315-
if _, exists, err := committeeCache.CommitteeCache.GetByKey(string(seed[:])); err == nil && exists {
315+
316+
if committeeCache.HasEntry(string(seed[:])) {
316317
return nil
317318
}
318319

‎beacon-chain/db/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ go_library(
2323
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/db",
2424
visibility = [
2525
"//beacon-chain:__subpackages__",
26+
"//fuzz:__pkg__",
2627
"//tools:__subpackages__",
2728
],
2829
deps = [

‎beacon-chain/forkchoice/protoarray/BUILD.bazel

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ go_library(
1414
"types.go",
1515
],
1616
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray",
17-
visibility = ["//beacon-chain:__subpackages__"],
17+
visibility = [
18+
"//beacon-chain:__subpackages__",
19+
"//fuzz:__pkg__",
20+
],
1821
deps = [
1922
"//shared/params:go_default_library",
2023
"@com_github_pkg_errors//:go_default_library",

‎beacon-chain/operations/attestations/BUILD.bazel

+4-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ go_library(
1212
"service.go",
1313
],
1414
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations",
15-
visibility = ["//beacon-chain:__subpackages__"],
15+
visibility = [
16+
"//beacon-chain:__subpackages__",
17+
"//fuzz:__pkg__",
18+
],
1619
deps = [
1720
"//beacon-chain/operations/attestations/kv:go_default_library",
1821
"//beacon-chain/state:go_default_library",

‎beacon-chain/operations/slashings/BUILD.bazel

+4-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,10 @@ go_library(
1111
"types.go",
1212
],
1313
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/operations/slashings",
14-
visibility = ["//beacon-chain:__subpackages__"],
14+
visibility = [
15+
"//beacon-chain:__subpackages__",
16+
"//fuzz:__pkg__",
17+
],
1518
deps = [
1619
"//beacon-chain/core/blocks:go_default_library",
1720
"//beacon-chain/core/helpers:go_default_library",

‎beacon-chain/operations/voluntaryexits/BUILD.bazel

+4-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ go_library(
88
"service.go",
99
],
1010
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits",
11-
visibility = ["//beacon-chain:__subpackages__"],
11+
visibility = [
12+
"//beacon-chain:__subpackages__",
13+
"//fuzz:__pkg__",
14+
],
1215
deps = [
1316
"//beacon-chain/core/helpers:go_default_library",
1417
"//beacon-chain/state:go_default_library",

‎beacon-chain/p2p/encoder/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ go_library(
1212
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/p2p/encoder",
1313
visibility = [
1414
"//beacon-chain:__subpackages__",
15+
"//fuzz:__pkg__",
1516
],
1617
deps = [
1718
"//shared/params:go_default_library",

‎beacon-chain/p2p/testing/BUILD.bazel

+5-1
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,17 @@ go_library(
44
name = "go_default_library",
55
testonly = True,
66
srcs = [
7+
"fuzz_p2p.go",
78
"mock_broadcaster.go",
89
"mock_peermanager.go",
910
"mock_peersprovider.go",
1011
"p2p.go",
1112
],
1213
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/p2p/testing",
13-
visibility = ["//beacon-chain:__subpackages__"],
14+
visibility = [
15+
"//beacon-chain:__subpackages__",
16+
"//fuzz:__pkg__",
17+
],
1418
deps = [
1519
"//beacon-chain/p2p/encoder:go_default_library",
1620
"//beacon-chain/p2p/peers:go_default_library",

0 commit comments

Comments
 (0)