diff --git a/chain/actors/adt/diff/generic.go b/chain/actors/adt/diff/generic.go index b3fa72926..4ae4aa054 100644 --- a/chain/actors/adt/diff/generic.go +++ b/chain/actors/adt/diff/generic.go @@ -28,6 +28,7 @@ type ArrayDiffer interface { // - All values that exist in curArr nnd not in prevArr are passed to ArrayDiffer.Add() // - All values that exist in preArr and in curArr are passed to ArrayDiffer.Modify() // - It is the responsibility of ArrayDiffer.Modify() to determine if the values it was passed have been modified. +// // If `preArr` and `curArr` are both backed by /v3/AMTs with the same bitwidth use the more efficient Amt method. func CompareArray(preArr, curArr adt.Array, out ArrayDiffer) error { notNew := make(map[int64]struct{}, curArr.Length()) @@ -84,6 +85,7 @@ type MapDiffer interface { // - All values that exist in curMap nnd not in prevArr are passed to MapDiffer.Add() // - All values that exist in preMap and in curMap are passed to MapDiffer.Modify() // - It is the responsibility of ArrayDiffer.Modify() to determine if the values it was passed have been modified. +// // If `preMap` and `curMap` are both backed by /v3/HAMTs with the same bitwidth and hash function use the more efficient Hamt method. func CompareMap(preMap, curMap adt.Map, out MapDiffer) error { notNew := make(map[string]struct{}) diff --git a/chain/actors/builtin/datacap/actor.go.template b/chain/actors/builtin/datacap/actor.go.template index 82b00e27b..012fc7f43 100644 --- a/chain/actors/builtin/datacap/actor.go.template +++ b/chain/actors/builtin/datacap/actor.go.template @@ -64,6 +64,11 @@ type State interface { VerifiedClients() (adt.Map, error) VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + + AllowanceMap() (adt.Map, error) + AllowanceMapBitWidth() int + AllowanceMapHashFunction() func(input []byte) []byte + AllowanceMapForOwner(owner address.Address) (adt.Map, error) } func AllCodes() []cid.Cid { diff --git a/chain/actors/builtin/datacap/datacap.go b/chain/actors/builtin/datacap/datacap.go index b71cd602d..fea665cac 100644 --- a/chain/actors/builtin/datacap/datacap.go +++ b/chain/actors/builtin/datacap/datacap.go @@ -10,10 +10,11 @@ import ( "github.com/filecoin-project/go-state-types/cbor" "github.com/ipfs/go-cid" - "github.com/filecoin-project/lily/chain/actors" lotusactors "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors" ) var ( @@ -65,6 +66,11 @@ type State interface { VerifiedClients() (adt.Map, error) VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + + AllowanceMap() (adt.Map, error) + AllowanceMapBitWidth() int + AllowanceMapHashFunction() func(input []byte) []byte + AllowanceMapForOwner(owner address.Address) (adt.Map, error) } func AllCodes() []cid.Cid { diff --git a/chain/actors/builtin/datacap/state.go.template b/chain/actors/builtin/datacap/state.go.template index e45b65390..94c4226f9 100644 --- a/chain/actors/builtin/datacap/state.go.template +++ b/chain/actors/builtin/datacap/state.go.template @@ -74,6 +74,30 @@ func (s *state{{.v}}) VerifiedClientsMapHashFunction() func(input []byte) []byte } } +func (s *state{{.v}}) AllowanceMap() (adt.Map, error) { + return adt{{.v}}.AsMap(s.store, s.Token.Allowances, int(s.Token.HamtBitWidth)) +} + +func (s *state{{.v}}) AllowanceMapBitWidth() int { + return int(s.Token.HamtBitWidth) +} + +func (s *state{{.v}}) AllowanceMapHashFunction() func(input []byte) []byte { + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } +} + +func (s *state{{.v}}) AllowanceMapForOwner(owner address.Address) (adt.Map, error) { + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(owner), s.Token.Allowances, int(s.Token.HamtBitWidth)) + if err != nil { + return nil, err + } + return adt{{.v}}.AsMap(s.store, innerHamtCid, int(s.Token.HamtBitWidth)) +} + + func (s *state{{.v}}) ActorKey() string { return actors.DatacapKey } @@ -90,3 +114,19 @@ func (s *state{{.v}}) Code() cid.Cid { return code } + +func (s *state{{.v}}) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + actorToHamtMap, err := adt{{.v}}.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil +} diff --git a/chain/actors/builtin/datacap/v9.go b/chain/actors/builtin/datacap/v9.go index 90ab8ed98..ce6ad13d4 100644 --- a/chain/actors/builtin/datacap/v9.go +++ b/chain/actors/builtin/datacap/v9.go @@ -7,10 +7,12 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors" + datacap9 "github.com/filecoin-project/go-state-types/builtin/v9/datacap" adt9 "github.com/filecoin-project/go-state-types/builtin/v9/util/adt" ) @@ -74,6 +76,29 @@ func (s *state9) VerifiedClientsMapHashFunction() func(input []byte) []byte { } } +func (s *state9) AllowanceMap() (adt.Map, error) { + return adt9.AsMap(s.store, s.Token.Allowances, int(s.Token.HamtBitWidth)) +} + +func (s *state9) AllowanceMapBitWidth() int { + return int(s.Token.HamtBitWidth) +} + +func (s *state9) AllowanceMapHashFunction() func(input []byte) []byte { + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } +} + +func (s *state9) AllowanceMapForOwner(owner address.Address) (adt.Map, error) { + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(owner), s.Token.Allowances, int(s.Token.HamtBitWidth)) + if err != nil { + return nil, err + } + return adt9.AsMap(s.store, innerHamtCid, int(s.Token.HamtBitWidth)) +} + func (s *state9) ActorKey() string { return actors.DatacapKey } @@ -90,3 +115,19 @@ func (s *state9) Code() cid.Cid { return code } + +func (s *state9) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + actorToHamtMap, err := adt9.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil +} diff --git a/chain/actors/builtin/market/actor.go.template b/chain/actors/builtin/market/actor.go.template index 36053de4d..46c94fe50 100644 --- a/chain/actors/builtin/market/actor.go.template +++ b/chain/actors/builtin/market/actor.go.template @@ -80,7 +80,7 @@ type DealStates interface { ForEach(cb func(id abi.DealID, ds DealState) error) error Get(id abi.DealID) (*DealState, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*DealState, error) } @@ -88,7 +88,7 @@ type DealProposals interface { ForEach(cb func(id abi.DealID, dp markettypes.DealProposal) error) error Get(id abi.DealID) (*markettypes.DealProposal, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*markettypes.DealProposal, error) } diff --git a/chain/actors/builtin/market/diff.go b/chain/actors/builtin/market/diff.go index 283e29d09..a53e4aa06 100644 --- a/chain/actors/builtin/market/diff.go +++ b/chain/actors/builtin/market/diff.go @@ -34,13 +34,13 @@ func DiffDealProposals(ctx context.Context, store adt.Store, pre, cur State) (*D diffContainer := NewMarketProposalsDiffContainer(preP, curP) if requiresLegacyDiffing(pre, cur, preOpts, curOpts) { log.Warn("actor AMT opts differ, running slower generic array diff", "preCID", pre.Code(), "curCID", cur.Code()) - if err := diff.CompareArray(preP.array(), curP.array(), diffContainer); err != nil { + if err := diff.CompareArray(preP.AsArray(), curP.AsArray(), diffContainer); err != nil { return nil, fmt.Errorf("diffing deal states: %w", err) } return diffContainer.Results, nil } - changes, err := diff.Amt(ctx, preP.array(), curP.array(), store, store, amt.UseTreeBitWidth(uint(preOpts))) + changes, err := diff.Amt(ctx, preP.AsArray(), curP.AsArray(), store, store, amt.UseTreeBitWidth(uint(preOpts))) if err != nil { return nil, err } @@ -116,13 +116,13 @@ func DiffDealStates(ctx context.Context, store adt.Store, pre, cur State) (*Deal diffContainer := NewMarketStatesDiffContainer(preS, curS) if requiresLegacyDiffing(pre, cur, preOpts, curOpts) { log.Warn("actor AMT opts differ, running slower generic array diff", "preCID", pre.Code(), "curCID", cur.Code()) - if err := diff.CompareArray(preS.array(), curS.array(), diffContainer); err != nil { + if err := diff.CompareArray(preS.AsArray(), curS.AsArray(), diffContainer); err != nil { return nil, fmt.Errorf("diffing deal states: %w", err) } return diffContainer.Results, nil } - changes, err := diff.Amt(ctx, preS.array(), curS.array(), store, store, amt.UseTreeBitWidth(uint(preOpts))) + changes, err := diff.Amt(ctx, preS.AsArray(), curS.AsArray(), store, store, amt.UseTreeBitWidth(uint(preOpts))) if err != nil { return nil, err } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 7c765b1e1..ccb3ef7e3 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -105,7 +105,7 @@ type DealStates interface { ForEach(cb func(id abi.DealID, ds DealState) error) error Get(id abi.DealID) (*DealState, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*DealState, error) } @@ -113,7 +113,7 @@ type DealProposals interface { ForEach(cb func(id abi.DealID, dp markettypes.DealProposal) error) error Get(id abi.DealID) (*markettypes.DealProposal, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*markettypes.DealProposal, error) } diff --git a/chain/actors/builtin/market/state.go.template b/chain/actors/builtin/market/state.go.template index 0ddee8698..5dce97966 100644 --- a/chain/actors/builtin/market/state.go.template +++ b/chain/actors/builtin/market/state.go.template @@ -134,7 +134,7 @@ func (s *dealStates{{.v}}) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates{{.v}}) array() adt.Array { +func (s *dealStates{{.v}}) AsArray() adt.Array { return s.Array } @@ -199,7 +199,7 @@ func (s *dealProposals{{.v}}) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals{{.v}}) array() adt.Array { +func (s *dealProposals{{.v}}) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 20bf51c9a..63db494e8 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -121,7 +121,7 @@ func (s *dealStates0) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates0) array() adt.Array { +func (s *dealStates0) AsArray() adt.Array { return s.Array } @@ -184,7 +184,7 @@ func (s *dealProposals0) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals0) array() adt.Array { +func (s *dealProposals0) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v2.go b/chain/actors/builtin/market/v2.go index 81bb6f87d..5dffdf854 100644 --- a/chain/actors/builtin/market/v2.go +++ b/chain/actors/builtin/market/v2.go @@ -121,7 +121,7 @@ func (s *dealStates2) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates2) array() adt.Array { +func (s *dealStates2) AsArray() adt.Array { return s.Array } @@ -184,7 +184,7 @@ func (s *dealProposals2) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals2) array() adt.Array { +func (s *dealProposals2) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v3.go b/chain/actors/builtin/market/v3.go index dfba355e4..1b6292c96 100644 --- a/chain/actors/builtin/market/v3.go +++ b/chain/actors/builtin/market/v3.go @@ -116,7 +116,7 @@ func (s *dealStates3) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates3) array() adt.Array { +func (s *dealStates3) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals3) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals3) array() adt.Array { +func (s *dealProposals3) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v4.go b/chain/actors/builtin/market/v4.go index c1002dcc6..119e219da 100644 --- a/chain/actors/builtin/market/v4.go +++ b/chain/actors/builtin/market/v4.go @@ -116,7 +116,7 @@ func (s *dealStates4) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates4) array() adt.Array { +func (s *dealStates4) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals4) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals4) array() adt.Array { +func (s *dealProposals4) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v5.go b/chain/actors/builtin/market/v5.go index c282dc842..af231d062 100644 --- a/chain/actors/builtin/market/v5.go +++ b/chain/actors/builtin/market/v5.go @@ -116,7 +116,7 @@ func (s *dealStates5) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates5) array() adt.Array { +func (s *dealStates5) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals5) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals5) array() adt.Array { +func (s *dealProposals5) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v6.go b/chain/actors/builtin/market/v6.go index 60355dc59..b4c8d104e 100644 --- a/chain/actors/builtin/market/v6.go +++ b/chain/actors/builtin/market/v6.go @@ -116,7 +116,7 @@ func (s *dealStates6) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates6) array() adt.Array { +func (s *dealStates6) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals6) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals6) array() adt.Array { +func (s *dealProposals6) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v7.go b/chain/actors/builtin/market/v7.go index 24f78821e..22a266cd4 100644 --- a/chain/actors/builtin/market/v7.go +++ b/chain/actors/builtin/market/v7.go @@ -116,7 +116,7 @@ func (s *dealStates7) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates7) array() adt.Array { +func (s *dealStates7) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals7) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals7) array() adt.Array { +func (s *dealProposals7) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v8.go b/chain/actors/builtin/market/v8.go index 42be7995c..b47be1e64 100644 --- a/chain/actors/builtin/market/v8.go +++ b/chain/actors/builtin/market/v8.go @@ -117,7 +117,7 @@ func (s *dealStates8) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates8) array() adt.Array { +func (s *dealStates8) AsArray() adt.Array { return s.Array } @@ -180,7 +180,7 @@ func (s *dealProposals8) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals8) array() adt.Array { +func (s *dealProposals8) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v9.go b/chain/actors/builtin/market/v9.go index 36586b69e..b613f09bc 100644 --- a/chain/actors/builtin/market/v9.go +++ b/chain/actors/builtin/market/v9.go @@ -117,7 +117,7 @@ func (s *dealStates9) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates9) array() adt.Array { +func (s *dealStates9) AsArray() adt.Array { return s.Array } @@ -175,7 +175,7 @@ func (s *dealProposals9) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals9) array() adt.Array { +func (s *dealProposals9) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/miner/actor.go.template b/chain/actors/builtin/miner/actor.go.template index 5d7c84bb6..4ade99414 100644 --- a/chain/actors/builtin/miner/actor.go.template +++ b/chain/actors/builtin/miner/actor.go.template @@ -105,6 +105,7 @@ type State interface { DeadlinesChanged(State) (bool, error) Info() (MinerInfo, error) + InfoCid() cid.Cid MinerInfoChanged(State) (bool, error) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) diff --git a/chain/actors/builtin/miner/diff_deferred.go b/chain/actors/builtin/miner/diff_deferred.go new file mode 100644 index 000000000..1e934b18a --- /dev/null +++ b/chain/actors/builtin/miner/diff_deferred.go @@ -0,0 +1,95 @@ +package miner + +import ( + "context" + + "github.com/filecoin-project/go-amt-ipld/v4" + "github.com/filecoin-project/go-hamt-ipld/v3" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/adt/diff" +) + +type Changes struct { + Added []*typegen.Deferred + Modified []*ChangeDiff + Removed []*typegen.Deferred +} + +type ChangeDiff struct { + Previous *typegen.Deferred + Current *typegen.Deferred +} + +// TODO there may be value in storing the map and array key in the change. + +func DiffPreCommitsDeferred(ctx context.Context, store adt.Store, parent, child State) (*Changes, error) { + parentMap, err := parent.PrecommitsMap() + if err != nil { + return nil, err + } + childMap, err := child.PrecommitsMap() + if err != nil { + return nil, err + } + // TODO handle case where bitwidths or hash functions differ. + changes, err := diff.Hamt(ctx, parentMap, childMap, store, store, hamt.UseHashFunction(parent.PrecommitsMapHashFunction()), hamt.UseTreeBitWidth(parent.PrecommitsMapBitWidth())) + if err != nil { + return nil, err + } + out := &Changes{ + Added: make([]*typegen.Deferred, 0, len(changes)), + Modified: make([]*ChangeDiff, 0, len(changes)), + Removed: make([]*typegen.Deferred, 0, len(changes)), + } + for _, change := range changes { + switch change.Type { + case hamt.Add: + out.Added = append(out.Added, change.After) + case hamt.Modify: + out.Modified = append(out.Modified, &ChangeDiff{ + Previous: change.Before, + Current: change.After, + }) + case hamt.Remove: + out.Removed = append(out.Removed, change.Before) + } + } + return out, nil +} + +func DiffSectorsDeferred(ctx context.Context, store adt.Store, parent, child State) (*Changes, error) { + parentArray, err := parent.SectorsArray() + if err != nil { + return nil, err + } + childArray, err := child.SectorsArray() + if err != nil { + return nil, err + } + // TODO handle case where bitwidths differ. + changes, err := diff.Amt(ctx, parentArray, childArray, store, store, amt.UseTreeBitWidth(uint(parent.SectorsAmtBitwidth()))) + if err != nil { + return nil, err + } + out := &Changes{ + Added: make([]*typegen.Deferred, 0, len(changes)), + Modified: make([]*ChangeDiff, 0, len(changes)), + Removed: make([]*typegen.Deferred, 0, len(changes)), + } + for _, change := range changes { + switch change.Type { + case amt.Add: + out.Added = append(out.Added, change.After) + case amt.Modify: + out.Modified = append(out.Modified, &ChangeDiff{ + Previous: change.Before, + Current: change.After, + }) + case amt.Remove: + out.Removed = append(out.Removed, change.Before) + } + } + return out, nil +} diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index c71ce8a93..c07876bf9 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -154,6 +154,7 @@ type State interface { DeadlinesChanged(State) (bool, error) Info() (MinerInfo, error) + InfoCid() cid.Cid MinerInfoChanged(State) (bool, error) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) diff --git a/chain/actors/builtin/miner/state.go.template b/chain/actors/builtin/miner/state.go.template index 7d8c4b967..48278794c 100644 --- a/chain/actors/builtin/miner/state.go.template +++ b/chain/actors/builtin/miner/state.go.template @@ -705,3 +705,7 @@ func (s *state{{.v}}) Code() cid.Cid { return code } + +func (s *state{{.v}}) InfoCid() cid.Cid { + return s.State.Info +} \ No newline at end of file diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index eca0f28d1..6d28bb2b8 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -611,3 +611,7 @@ func (s *state0) Code() cid.Cid { return code } + +func (s *state0) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 2fd9f6102..d4ac71ee7 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -641,3 +641,7 @@ func (s *state2) Code() cid.Cid { return code } + +func (s *state2) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v3.go b/chain/actors/builtin/miner/v3.go index b07ab6487..867f0ff4f 100644 --- a/chain/actors/builtin/miner/v3.go +++ b/chain/actors/builtin/miner/v3.go @@ -642,3 +642,7 @@ func (s *state3) Code() cid.Cid { return code } + +func (s *state3) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v4.go b/chain/actors/builtin/miner/v4.go index 5a60a2dc0..0982bac3b 100644 --- a/chain/actors/builtin/miner/v4.go +++ b/chain/actors/builtin/miner/v4.go @@ -642,3 +642,7 @@ func (s *state4) Code() cid.Cid { return code } + +func (s *state4) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v5.go b/chain/actors/builtin/miner/v5.go index 15e7c53ed..646724c18 100644 --- a/chain/actors/builtin/miner/v5.go +++ b/chain/actors/builtin/miner/v5.go @@ -642,3 +642,7 @@ func (s *state5) Code() cid.Cid { return code } + +func (s *state5) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v6.go b/chain/actors/builtin/miner/v6.go index ec4d81dfe..94a0fa171 100644 --- a/chain/actors/builtin/miner/v6.go +++ b/chain/actors/builtin/miner/v6.go @@ -642,3 +642,7 @@ func (s *state6) Code() cid.Cid { return code } + +func (s *state6) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v7.go b/chain/actors/builtin/miner/v7.go index ea5f39316..95b483e67 100644 --- a/chain/actors/builtin/miner/v7.go +++ b/chain/actors/builtin/miner/v7.go @@ -643,3 +643,7 @@ func (s *state7) Code() cid.Cid { return code } + +func (s *state7) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v8.go b/chain/actors/builtin/miner/v8.go index b33d7fa09..81c1feaa3 100644 --- a/chain/actors/builtin/miner/v8.go +++ b/chain/actors/builtin/miner/v8.go @@ -642,3 +642,7 @@ func (s *state8) Code() cid.Cid { return code } + +func (s *state8) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v9.go b/chain/actors/builtin/miner/v9.go index 26f930e48..79aaaa7ce 100644 --- a/chain/actors/builtin/miner/v9.go +++ b/chain/actors/builtin/miner/v9.go @@ -606,3 +606,7 @@ func (s *state9) Code() cid.Cid { return code } + +func (s *state9) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/verifreg/actor.go.template b/chain/actors/builtin/verifreg/actor.go.template index 6882b000a..dfffe00fe 100644 --- a/chain/actors/builtin/verifreg/actor.go.template +++ b/chain/actors/builtin/verifreg/actor.go.template @@ -72,6 +72,16 @@ type State interface { VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + ClaimsMap() (adt.Map, error) + ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) + ClaimsMapBitWidth() int + ClaimsMapHashFunction() func(input []byte) []byte + + AllocationsMap() (adt.Map, error) + AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) + AllocationsMapBitWidth() int + AllocationsMapHashFunction() func(input []byte) []byte + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) diff --git a/chain/actors/builtin/verifreg/state.go.template b/chain/actors/builtin/verifreg/state.go.template index 0a2512b3d..c6c8eab33 100644 --- a/chain/actors/builtin/verifreg/state.go.template +++ b/chain/actors/builtin/verifreg/state.go.template @@ -8,6 +8,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" +{{if (ge .v 9)}} + cbg "github.com/whyrusleeping/cbor-gen" +{{end}} + "github.com/filecoin-project/lily/chain/actors/adt" {{if (le .v 1)}} @@ -126,6 +130,48 @@ func (s *state{{.v}}) VerifiersMapHashFunction() func(input []byte) []byte { {{end}} } +func (s *state{{.v}}) AllocationsMap() (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + return adt{{.v}}.AsMap(s.store, s.Allocations{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}}) +{{end}} +} + +func (s *state{{.v}}) AllocationsMapBitWidth() int { +{{if (ge .v 3)}} + return builtin{{.v}}.DefaultHamtBitwidth +{{else}} + return 5 +{{end}} +} + +func (s *state{{.v}}) AllocationsMapHashFunction() func(input []byte) []byte { +{{if (le .v 1)}} + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } +{{else}} + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } +{{end}} +} + +func (s *state{{.v}}) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(clientIdAddr), s.Allocations, builtin{{.v}}.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt{{.v}}.AsMap(s.store, innerHamtCid, builtin{{.v}}.DefaultHamtBitwidth) +{{end}} +} + func (s *state{{.v}}) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -198,3 +244,68 @@ func (s *state{{.v}}) GetClaims(providerIdAddr address.Address) (map[verifreg9.C return s.LoadClaimsToMap(s.store, providerIdAddr) {{end}} } + +func (s *state{{.v}}) ClaimsMap() (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + return adt{{.v}}.AsMap(s.store, s.Claims, builtin{{.v}}.DefaultHamtBitwidth) +{{end}} +} + +// TODO this could return an error since not all versions have a claims map +func (s *state{{.v}}) ClaimsMapBitWidth() int { + {{if (ge .v 3)}} + return builtin{{.v}}.DefaultHamtBitwidth + {{else}} + return 5 + {{end}} +} + +// TODO this could return an error since not all versions have a claims map +func (s *state{{.v}}) ClaimsMapHashFunction() func(input []byte) []byte { + {{if (le .v 1)}} + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } + {{else}} + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + {{end}} +} + +func (s *state{{.v}}) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(providerIdAddr), s.Claims, builtin{{.v}}.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt{{.v}}.AsMap(s.store, innerHamtCid, builtin{{.v}}.DefaultHamtBitwidth) +{{end}} +} + +func (s *state{{.v}}) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { +{{if (le .v 8)}} + return cid.Undef, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + actorToHamtMap, err := adt{{.v}}.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil +{{end}} +} + diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index c48716228..26f760c7b 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -93,6 +93,33 @@ func (s *state0) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state0) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + +func (s *state0) AllocationsMapBitWidth() int { + + return 5 + +} + +func (s *state0) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } + +} + +func (s *state0) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + func (s *state0) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -153,3 +180,38 @@ func (s *state0) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v0") } + +func (s *state0) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state0) ClaimsMapBitWidth() int { + + return 5 + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state0) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } + +} + +func (s *state0) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + +func (s *state0) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v0") + +} diff --git a/chain/actors/builtin/verifreg/v2.go b/chain/actors/builtin/verifreg/v2.go index a1d388436..f505b5bb9 100644 --- a/chain/actors/builtin/verifreg/v2.go +++ b/chain/actors/builtin/verifreg/v2.go @@ -93,6 +93,33 @@ func (s *state2) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state2) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + +func (s *state2) AllocationsMapBitWidth() int { + + return 5 + +} + +func (s *state2) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state2) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + func (s *state2) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -153,3 +180,38 @@ func (s *state2) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v2") } + +func (s *state2) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state2) ClaimsMapBitWidth() int { + + return 5 + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state2) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state2) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + +func (s *state2) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v2") + +} diff --git a/chain/actors/builtin/verifreg/v3.go b/chain/actors/builtin/verifreg/v3.go index 3be489f1f..7e3c15178 100644 --- a/chain/actors/builtin/verifreg/v3.go +++ b/chain/actors/builtin/verifreg/v3.go @@ -95,6 +95,33 @@ func (s *state3) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state3) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + +func (s *state3) AllocationsMapBitWidth() int { + + return builtin3.DefaultHamtBitwidth + +} + +func (s *state3) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state3) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + func (s *state3) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -155,3 +182,38 @@ func (s *state3) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v3") } + +func (s *state3) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state3) ClaimsMapBitWidth() int { + + return builtin3.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state3) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state3) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + +func (s *state3) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v3") + +} diff --git a/chain/actors/builtin/verifreg/v4.go b/chain/actors/builtin/verifreg/v4.go index 819b547ad..0e7ab3fb6 100644 --- a/chain/actors/builtin/verifreg/v4.go +++ b/chain/actors/builtin/verifreg/v4.go @@ -95,6 +95,33 @@ func (s *state4) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state4) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + +func (s *state4) AllocationsMapBitWidth() int { + + return builtin4.DefaultHamtBitwidth + +} + +func (s *state4) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state4) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + func (s *state4) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -155,3 +182,38 @@ func (s *state4) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v4") } + +func (s *state4) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state4) ClaimsMapBitWidth() int { + + return builtin4.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state4) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state4) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + +func (s *state4) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v4") + +} diff --git a/chain/actors/builtin/verifreg/v5.go b/chain/actors/builtin/verifreg/v5.go index 83a2496f5..f701c09ff 100644 --- a/chain/actors/builtin/verifreg/v5.go +++ b/chain/actors/builtin/verifreg/v5.go @@ -95,6 +95,33 @@ func (s *state5) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state5) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + +func (s *state5) AllocationsMapBitWidth() int { + + return builtin5.DefaultHamtBitwidth + +} + +func (s *state5) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state5) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + func (s *state5) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -155,3 +182,38 @@ func (s *state5) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v5") } + +func (s *state5) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state5) ClaimsMapBitWidth() int { + + return builtin5.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state5) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state5) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + +func (s *state5) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v5") + +} diff --git a/chain/actors/builtin/verifreg/v6.go b/chain/actors/builtin/verifreg/v6.go index 8aec727be..851cda196 100644 --- a/chain/actors/builtin/verifreg/v6.go +++ b/chain/actors/builtin/verifreg/v6.go @@ -95,6 +95,33 @@ func (s *state6) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state6) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + +func (s *state6) AllocationsMapBitWidth() int { + + return builtin6.DefaultHamtBitwidth + +} + +func (s *state6) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state6) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + func (s *state6) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -155,3 +182,38 @@ func (s *state6) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v6") } + +func (s *state6) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state6) ClaimsMapBitWidth() int { + + return builtin6.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state6) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state6) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + +func (s *state6) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v6") + +} diff --git a/chain/actors/builtin/verifreg/v7.go b/chain/actors/builtin/verifreg/v7.go index 36b157b93..4e53e548b 100644 --- a/chain/actors/builtin/verifreg/v7.go +++ b/chain/actors/builtin/verifreg/v7.go @@ -95,6 +95,33 @@ func (s *state7) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state7) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + +func (s *state7) AllocationsMapBitWidth() int { + + return builtin7.DefaultHamtBitwidth + +} + +func (s *state7) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state7) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + func (s *state7) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -154,3 +181,38 @@ func (s *state7) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v7") } + +func (s *state7) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state7) ClaimsMapBitWidth() int { + + return builtin7.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state7) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state7) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + +func (s *state7) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v7") + +} diff --git a/chain/actors/builtin/verifreg/v8.go b/chain/actors/builtin/verifreg/v8.go index ab2ade637..3e0b4d483 100644 --- a/chain/actors/builtin/verifreg/v8.go +++ b/chain/actors/builtin/verifreg/v8.go @@ -94,6 +94,33 @@ func (s *state8) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state8) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + +func (s *state8) AllocationsMapBitWidth() int { + + return builtin8.DefaultHamtBitwidth + +} + +func (s *state8) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state8) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + func (s *state8) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -153,3 +180,38 @@ func (s *state8) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v8") } + +func (s *state8) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state8) ClaimsMapBitWidth() int { + + return builtin8.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state8) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state8) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + +func (s *state8) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v8") + +} diff --git a/chain/actors/builtin/verifreg/v9.go b/chain/actors/builtin/verifreg/v9.go index 3b8655b81..9bafc6003 100644 --- a/chain/actors/builtin/verifreg/v9.go +++ b/chain/actors/builtin/verifreg/v9.go @@ -8,6 +8,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lily/chain/actors/adt" "crypto/sha256" @@ -94,6 +96,37 @@ func (s *state9) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state9) AllocationsMap() (adt.Map, error) { + + return adt9.AsMap(s.store, s.Allocations, builtin9.DefaultHamtBitwidth) + +} + +func (s *state9) AllocationsMapBitWidth() int { + + return builtin9.DefaultHamtBitwidth + +} + +func (s *state9) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state9) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(clientIdAddr), s.Allocations, builtin9.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt9.AsMap(s.store, innerHamtCid, builtin9.DefaultHamtBitwidth) + +} + func (s *state9) RootKey() (address.Address, error) { return s.State.RootKey, nil } @@ -153,3 +186,54 @@ func (s *state9) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return s.LoadClaimsToMap(s.store, providerIdAddr) } + +func (s *state9) ClaimsMap() (adt.Map, error) { + + return adt9.AsMap(s.store, s.Claims, builtin9.DefaultHamtBitwidth) + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state9) ClaimsMapBitWidth() int { + + return builtin9.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state9) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state9) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(providerIdAddr), s.Claims, builtin9.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt9.AsMap(s.store, innerHamtCid, builtin9.DefaultHamtBitwidth) + +} + +func (s *state9) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + actorToHamtMap, err := adt9.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil + +} diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 104ff2d4e..ef2684a07 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -99,6 +99,16 @@ type State interface { VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + ClaimsMap() (adt.Map, error) + ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) + ClaimsMapBitWidth() int + ClaimsMapHashFunction() func(input []byte) []byte + + AllocationsMap() (adt.Map, error) + AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) + AllocationsMapBitWidth() int + AllocationsMapHashFunction() func(input []byte) []byte + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) diff --git a/chain/datasource/datasource.go b/chain/datasource/datasource.go index cb2a27a46..733337bfc 100644 --- a/chain/datasource/datasource.go +++ b/chain/datasource/datasource.go @@ -138,6 +138,22 @@ type DataSource struct { diffPreCommitGroup singleflight.Group } +func (t *DataSource) NetworkName(ctx context.Context) (string, error) { + name, err := t.node.StateNetworkName(ctx) + if err != nil { + return "", err + } + return string(name), nil +} + +func (t *DataSource) NetworkVersion(ctx context.Context, tsk types.TipSetKey) (uint, error) { + v, err := t.node.StateNetworkVersion(ctx, tsk) + if err != nil { + return 0, err + } + return uint(v), nil +} + func (t *DataSource) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) { return t.node.ComputeBaseFee(ctx, ts) } @@ -238,6 +254,10 @@ func (t *DataSource) CirculatingSupply(ctx context.Context, ts *types.TipSet) (a return t.node.CirculatingSupply(ctx, ts.Key()) } +func (t *DataSource) MessageExecutionsV2(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecutionV2, error) { + return t.node.GetMessageExecutionsForTipSetV2(ctx, ts, pts) +} + func (t *DataSource) MessageExecutions(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecution, error) { metrics.RecordInc(ctx, metrics.DataSourceMessageExecutionRead) ctx, span := otel.Tracer("").Start(ctx, "DataSource.MessageExecutions") @@ -282,6 +302,10 @@ func (t *DataSource) ShouldBurnFn(ctx context.Context, ts *types.TipSet) (lens.S return t.node.BurnFundsFn(ctx, ts) } +func (t *DataSource) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { + return t.node.ChainReadObj(ctx, obj) +} + func ComputeGasOutputs(ctx context.Context, block *types.BlockHeader, message *types.Message, receipt *types.MessageReceipt, shouldBurnFn lens.ShouldBurnFn) (vm.GasOutputs, error) { burn, err := shouldBurnFn(ctx, message, receipt.ExitCode) if err != nil { diff --git a/chain/indexer/distributed/queue/tracing/carrier.go b/chain/indexer/distributed/queue/tracing/carrier.go index caf106460..0c95126d2 100644 --- a/chain/indexer/distributed/queue/tracing/carrier.go +++ b/chain/indexer/distributed/queue/tracing/carrier.go @@ -32,7 +32,7 @@ type TraceCarrier struct { Remote bool `json:"remote"` } -//MarshalJSON converts TraceCarrier to a trace.SpanContext and marshals it to JSON. +// MarshalJSON converts TraceCarrier to a trace.SpanContext and marshals it to JSON. func (c *TraceCarrier) MarshalJSON() ([]byte, error) { return c.AsSpanContext().MarshalJSON() } diff --git a/chain/indexer/tasktype/table_tasks.go b/chain/indexer/tasktype/table_tasks.go index 309dfe341..407cbbb78 100644 --- a/chain/indexer/tasktype/table_tasks.go +++ b/chain/indexer/tasktype/table_tasks.go @@ -197,7 +197,7 @@ var TableFieldComments = map[string]map[string]string{ "Label": "An arbitrary client chosen label to apply to the deal. The value is base64 encoded before persisting.", "PaddedPieceSize": "The piece size in bytes with padding.", "PieceCID": "CID of a sector piece. A Piece is an object that represents a whole or part of a File.", - "ProviderCollateral": "The amount of FIL (in attoFIL) the provider has pledged as collateral. The Provider deal collateral is only slashed when a sector is terminated before the deal expires.", + "ProviderCollateral": "The amount of FIL (in attoFIL) the provider has pledged as collateral. The Client deal collateral is only slashed when a sector is terminated before the deal expires.", "ProviderID": "Address of the actor providing the services.", "StartEpoch": "The epoch at which this deal with begin. Storage deal must appear in a sealed (proven) sector no later than start_epoch, otherwise it is invalid.", "StateRoot": "CID of the parent state root for this deal.", diff --git a/commands/job/index.go b/commands/job/index.go index 0a36fa877..54edb18e5 100644 --- a/commands/job/index.go +++ b/commands/job/index.go @@ -3,6 +3,7 @@ package job import ( "fmt" "strings" + "time" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" @@ -33,6 +34,7 @@ The index command may be used to index a single tipset from the filecoin blockch Subcommands: []*cli.Command{ IndexTipSetCmd, IndexHeightCmd, + IndexFileCmd, }, Before: func(cctx *cli.Context) error { tasks := RunFlags.Tasks.Value() @@ -47,11 +49,47 @@ The index command may be used to index a single tipset from the filecoin blockch }, } +var IndexFileCmd = &cli.Command{ + Name: "file", + Usage: "Index the state of a car file produced by lily.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Usage: "Name of the file to persist state to.", + Value: fmt.Sprintf("lily_state_%.car", time.Now().Unix()), + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lotuscli.ReqContext(cctx) + + api, closer, err := commands.GetAPI(ctx) + if err != nil { + return err + } + defer closer() + + cfg := &lily.LilyIndexIPLDFileConfig{ + JobConfig: RunFlags.ParseJobConfig("index-file"), + Path: cctx.String("filename"), + } + + success, err := api.LilyIndexIPLDFile(ctx, cfg) + fmt.Println("success: ", success) + if err != nil { + return err + } + + return nil + + }, +} + var IndexTipSetCmd = &cli.Command{ Name: "tipset", Usage: "Index the state of a tipset from the filecoin blockchain by tipset key.", Subcommands: []*cli.Command{ IndexNotifyCmd, + IndexIPLDCmd, }, Flags: []cli.Flag{ &cli.StringFlag{ @@ -107,6 +145,7 @@ var IndexHeightCmd = &cli.Command{ }, Subcommands: []*cli.Command{ IndexNotifyCmd, + IndexIPLDCmd, }, Before: func(cctx *cli.Context) error { ctx := lotuscli.ReqContext(cctx) @@ -150,6 +189,48 @@ var IndexHeightCmd = &cli.Command{ }, } +var IndexIPLDCmd = &cli.Command{ + Name: "ipld", + Usage: "Index the state of a tipset from the filecoin blockchain persisted as a car file.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Usage: "Name of the file to persist state to.", + Value: fmt.Sprintf("lily_state_%d.car", time.Now().Unix()), + }, + &cli.IntFlag{ + Name: "write-buffer", + Usage: "specify write buffer size", + Value: 1 << 20, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lotuscli.ReqContext(cctx) + + api, closer, err := commands.GetAPI(ctx) + if err != nil { + return err + } + defer closer() + + cfg := &lily.LilyIndexIPLDConfig{ + JobConfig: RunFlags.ParseJobConfig("index-ipld"), + TipSet: indexFlags.tipsetKey, + Path: cctx.String("filename"), + WriteBufferSize: cctx.Int("write-buffer"), + } + + success, err := api.LilyIndexIPLD(ctx, cfg) + fmt.Println("success: ", success) + if err != nil { + return err + } + + return nil + + }, +} + var IndexNotifyCmd = &cli.Command{ Name: "notify", Usage: "notify the provided queueing system of the tipset to index allowing tipset-workers to perform the indexing.", diff --git a/go.mod b/go.mod index 15e185fe3..25f62cf68 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/go-pg/pg/v10 v10.10.6 github.com/hashicorp/golang-lru v0.5.4 github.com/ipfs/go-block-format v0.0.3 - github.com/ipfs/go-cid v0.2.0 + github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-fs-lock v0.0.7 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipld-cbor v0.0.6 @@ -54,8 +54,8 @@ require ( go.uber.org/fx v1.15.0 go.uber.org/multierr v1.8.0 go.uber.org/zap v1.22.0 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/text v0.3.7 + golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 + golang.org/x/text v0.5.0 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f gopkg.in/cheggaaa/pb.v1 v1.0.28 ) @@ -72,6 +72,8 @@ require ( github.com/multiformats/go-varint v0.0.6 go.opentelemetry.io/otel/trace v1.7.0 go.uber.org/atomic v1.10.0 + gorm.io/driver/postgres v1.4.6 + gorm.io/gorm v1.24.3 ) require ( @@ -210,15 +212,19 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect github.com/ipld/go-car/v2 v2.4.1 // indirect github.com/ipld/go-codec-dagpb v1.3.2 // indirect - github.com/ipld/go-ipld-prime v0.17.0 // indirect + github.com/ipld/go-ipld-prime v0.19.0 // indirect github.com/ipld/go-ipld-selector-text-lite v0.0.1 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.2.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/jessevdk/go-flags v1.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect @@ -269,7 +275,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.5.0 // indirect + github.com/multiformats/go-multicodec v0.6.0 // indirect github.com/multiformats/go-multistream v0.3.3 // indirect github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c // indirect github.com/nkovacs/streamquote v1.0.0 // indirect @@ -319,12 +325,12 @@ require ( go.opentelemetry.io/otel/sdk/export/metric v0.25.0 // indirect go.uber.org/dig v1.12.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect - golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect + golang.org/x/crypto v0.4.0 // indirect golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/net v0.3.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect diff --git a/go.sum b/go.sum index 3c467edeb..b934af554 100644 --- a/go.sum +++ b/go.sum @@ -514,7 +514,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -655,8 +655,9 @@ github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnO github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= -github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= +github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -831,8 +832,8 @@ github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= -github.com/ipld/go-ipld-prime v0.17.0 h1:+U2peiA3aQsE7mrXjD2nYZaZrCcakoz2Wge8K42Ld8g= -github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= +github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= +github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= @@ -840,6 +841,14 @@ github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5d github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.2.0 h1:NdPpngX0Y6z6XDFKqmFQaE+bCtkqzvQIOt1wvBlAqs8= +github.com/jackc/pgx/v5 v5.2.0/go.mod h1:Ptn7zmohNsWEsdxRawMzk3gaKma2obW+NWTnKa0S4nk= +github.com/jackc/puddle/v2 v2.1.2/go.mod h1:2lpufsF5mRHO6SuZkm0fNYxM6SWHfvyFj62KwNzgels= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -863,6 +872,9 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -1343,8 +1355,8 @@ github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs= -github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= +github.com/multiformats/go-multicodec v0.6.0 h1:KhH2kSuCARyuJraYMFxrNO3DqIaYhOdS039kbhgVwpE= +github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1721,6 +1733,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 h1:O9XLFXGkVswDFmH9LaYpqu+r/AAFWqr0DL6V00KEVFg= github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= @@ -1857,8 +1870,9 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1970,8 +1984,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= -golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1993,8 +2008,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 h1:ZrnxWX62AgTKOSagEqxvb3ffipvEDX2pl7E1TdqLqIc= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2101,13 +2117,16 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2115,8 +2134,10 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2336,6 +2357,11 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.4.6 h1:1FPESNXqIKG5JmraaH2bfCVlMQ7paLoCreFxDtqzwdc= +gorm.io/driver/postgres v1.4.6/go.mod h1:UJChCNLFKeBqQRE+HrkFUbKbq9idPXmTOk2u4Wok8S4= +gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.3 h1:WL2ifUmzR/SLp85CSURAfybcHnGZ+yLSGSxgYXlFBHg= +gorm.io/gorm v1.24.3/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/lens/interface.go b/lens/interface.go index f6f267bb9..6e139d16b 100644 --- a/lens/interface.go +++ b/lens/interface.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/go-address" @@ -23,6 +24,7 @@ type API interface { VMAPI GetMessageExecutionsForTipSet(ctx context.Context, ts, pts *types.TipSet) ([]*MessageExecution, error) + GetMessageExecutionsForTipSetV2(ctx context.Context, ts, pts *types.TipSet) ([]*MessageExecutionV2, error) } type StoreAPI interface { // TODO this should be the lotus store not the specs-actors store. @@ -63,6 +65,7 @@ type StateAPI interface { StateGetReceipt(ctx context.Context, bcid cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) CirculatingSupply(context.Context, types.TipSetKey) (api.CirculatingSupply, error) StateNetworkName(context.Context) (dtypes.NetworkName, error) + StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) } type ShouldBurnFn func(ctx context.Context, msg *types.Message, errcode exitcode.ExitCode) (bool, error) @@ -85,6 +88,17 @@ type MessageExecution struct { Implicit bool } +type MessageExecutionV2 struct { + Cid cid.Cid + StateRoot cid.Cid + Height abi.ChainEpoch + + Message *types.Message + Ret *vm.ApplyRet + + Implicit bool +} + type BlockMessages struct { Block *types.BlockHeader // block messages appeared in BlsMessages []*types.Message // BLS messages in block `Block` diff --git a/lens/lily/api.go b/lens/lily/api.go index 2ccf8d47e..67779bd20 100644 --- a/lens/lily/api.go +++ b/lens/lily/api.go @@ -20,6 +20,8 @@ type LilyAPI interface { AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) LilyIndex(ctx context.Context, cfg *LilyIndexConfig) (interface{}, error) + LilyIndexIPLD(ctx context.Context, cfg *LilyIndexIPLDConfig) (bool, error) + LilyIndexIPLDFile(ctx context.Context, cfg *LilyIndexIPLDFileConfig) (bool, error) LilyWatch(ctx context.Context, cfg *LilyWatchConfig) (*schedule.JobSubmitResult, error) LilyWalk(ctx context.Context, cfg *LilyWalkConfig) (*schedule.JobSubmitResult, error) LilySurvey(ctx context.Context, cfg *LilySurveyConfig) (*schedule.JobSubmitResult, error) @@ -160,6 +162,20 @@ type LilyIndexConfig struct { TipSet types.TipSetKey } +type LilyIndexIPLDFileConfig struct { + JobConfig LilyJobConfig + + Path string +} + +type LilyIndexIPLDConfig struct { + JobConfig LilyJobConfig + + TipSet types.TipSetKey + Path string + WriteBufferSize int +} + type LilyIndexNotifyConfig struct { IndexConfig LilyIndexConfig diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 00ebe1eab..8160de0ac 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -3,6 +3,7 @@ package lily import ( "context" "fmt" + "os" "strconv" "sync" @@ -41,6 +42,9 @@ import ( "github.com/filecoin-project/lily/lens/lily/modules" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" + indexer2 "github.com/filecoin-project/lily/pkg/indexer" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -132,6 +136,94 @@ func (m *LilyNodeAPI) StartTipSetWorker(_ context.Context, cfg *LilyTipSetWorker return res, nil } +func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) (bool, error) { + return false, nil + /* + // the context's passed to these methods live for the duration of the clients request, so make a new one. + ctx := context.Background() + + // TODO add file buffering + f, err := os.OpenFile(cfg.Path, os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0o644) + defer f.Close() + if err != nil { + return false, err + } + + taskAPI, err := datasource.NewDataSource(m) + if err != nil { + return false, err + } + // connect to the database, were gonna write stuff to this. + db, err := gorm.Open(postgres.Open("host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable"), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list + // TODO figure out how to make a new schema with gorm... + }, + }) + if err != nil { + return false, err + } + if err := lambda.ParseParams(ctx, taskAPI, db); err != nil { + return false, err + } + + + currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) + if err != nil { + return false, err + } + + executedTs, err := m.ChainGetTipSet(ctx, currentTs.Parents()) + if err != nil { + return false, err + } + + chainState, err := extract.State(ctx, taskAPI, currentTs, executedTs) + if err != nil { + return false, err + } + + bs := blockstore.NewMemorySync() + if err := cbor.Persist(ctx, chainState); err != nil; { + return false, err + } + if err != nil { + return false, err + } + if err := cbor.WriteCarV1(ctx, root, bs, f); err != nil { + return false, err + } + return true, nil + */ +} + +func (m *LilyNodeAPI) LilyIndexIPLDFile(_ context.Context, cfg *LilyIndexIPLDFileConfig) (bool, error) { + // the context's passed to these methods live for the duration of the clients request, so make a new one. + ctx := context.Background() + md := storage.Metadata{ + JobName: cfg.JobConfig.Name, + } + + // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. + strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) + if err != nil { + return false, err + } + + // TODO add buffering + f, err := os.OpenFile(cfg.Path, os.O_RDONLY, 0o644) + defer f.Close() + if err != nil { + return false, err + } + + if err := timescale.Process(ctx, f, strg, m.StateManager.GetNetworkVersion); err != nil { + return false, err + } + + return true, nil +} + func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interface{}, error) { md := storage.Metadata{ JobName: cfg.JobConfig.Name, @@ -284,29 +376,58 @@ func (m *LilyNodeAPI) LilyWatchNotify(_ context.Context, cfg *LilyWatchNotifyCon } func (m *LilyNodeAPI) LilyWalk(_ context.Context, cfg *LilyWalkConfig) (*schedule.JobSubmitResult, error) { - // the context's passed to these methods live for the duration of the clients request, so make a new one. - ctx := context.Background() + /* + // the context's passed to these methods live for the duration of the clients request, so make a new one. + ctx := context.Background() - md := storage.Metadata{ - JobName: cfg.JobConfig.Name, - } + md := storage.Metadata{ + JobName: cfg.JobConfig.Name, + } - // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. - strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) - if err != nil { - return nil, err - } + // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. + strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) + if err != nil { + return nil, err + } + + taskAPI, err := datasource.NewDataSource(m) + if err != nil { + return nil, err + } + + // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. + idx, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + if err != nil { + return nil, err + } + + res := m.Scheduler.Submit(&schedule.JobConfig{ + Name: cfg.JobConfig.Name, + Type: "walk", + Params: map[string]string{ + "window": cfg.JobConfig.Window.String(), + "minHeight": fmt.Sprintf("%d", cfg.From), + "maxHeight": fmt.Sprintf("%d", cfg.To), + "storage": cfg.JobConfig.Storage, + }, + Tasks: cfg.JobConfig.Tasks, + Job: walk.NewWalker(idx, m, cfg.JobConfig.Name, cfg.JobConfig.Tasks, cfg.From, cfg.To), + RestartOnFailure: cfg.JobConfig.RestartOnFailure, + RestartOnCompletion: cfg.JobConfig.RestartOnCompletion, + RestartDelay: cfg.JobConfig.RestartDelay, + }) + + */ taskAPI, err := datasource.NewDataSource(m) if err != nil { return nil, err } - - // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. - idx, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + transformer, err := cbor.NewTransformer("/Users/frrist/Workspace/src/github.com/filecoin-project/lily/garage", cfg.JobConfig.Name) if err != nil { return nil, err } + stateIndexer := indexer2.NewStateIndexer(taskAPI, transformer) res := m.Scheduler.Submit(&schedule.JobConfig{ Name: cfg.JobConfig.Name, @@ -318,7 +439,7 @@ func (m *LilyNodeAPI) LilyWalk(_ context.Context, cfg *LilyWalkConfig) (*schedul "storage": cfg.JobConfig.Storage, }, Tasks: cfg.JobConfig.Tasks, - Job: walk.NewWalker(idx, m, cfg.JobConfig.Name, cfg.JobConfig.Tasks, cfg.From, cfg.To), + Job: walk.NewWalker(stateIndexer, m, cfg.JobConfig.Name, cfg.JobConfig.Tasks, cfg.From, cfg.To), RestartOnFailure: cfg.JobConfig.RestartOnFailure, RestartOnCompletion: cfg.JobConfig.RestartOnCompletion, RestartDelay: cfg.JobConfig.RestartDelay, @@ -538,6 +659,48 @@ func (m *LilyNodeAPI) GetMessageExecutionsForTipSet(ctx context.Context, next *t return out, nil } +func (m *LilyNodeAPI) GetMessageExecutionsForTipSetV2(ctx context.Context, next *types.TipSet, current *types.TipSet) ([]*lens.MessageExecutionV2, error) { + // this is defined in the lily daemon dep injection constructor, failure here is a developer error. + msgMonitor, ok := m.ExecMonitor.(*modules.BufferedExecMonitor) + if !ok { + panic(fmt.Sprintf("bad cast, developer error expected modules.BufferedExecMonitor, got %T", m.ExecMonitor)) + } + + // if lily was watching the chain when this tipset was applied then its exec monitor will already + // contain executions for this tipset. + executions, err := msgMonitor.ExecutionFor(current) + if err != nil { + if err == modules.ErrExecutionTraceNotFound { + // if lily hasn't watched this tipset be applied then we need to compute its execution trace. + // this will likely be the case for most walk tasks. + _, err := m.StateManager.ExecutionTraceWithMonitor(ctx, current, msgMonitor) + if err != nil { + return nil, fmt.Errorf("failed to compute execution trace for tipset %s: %w", current.Key().String(), err) + } + // the above call will populate the msgMonitor with an execution trace for this tipset, get it. + executions, err = msgMonitor.ExecutionFor(current) + if err != nil { + return nil, fmt.Errorf("failed to find execution trace for tipset %s: %w", current.Key().String(), err) + } + } else { + return nil, fmt.Errorf("failed to extract message execution for tipset %s: %w", next, err) + } + } + + out := make([]*lens.MessageExecutionV2, len(executions)) + for idx, execution := range executions { + out[idx] = &lens.MessageExecutionV2{ + Cid: execution.Mcid, + StateRoot: execution.TipSet.ParentState(), + Height: execution.TipSet.Height(), + Message: execution.Msg, + Ret: execution.Ret, + Implicit: execution.Implicit, + } + } + return out, nil +} + // ComputeBaseFee calculates the base-fee of the specified tipset. func (m *LilyNodeAPI) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) { return m.ChainAPI.Chain.ComputeBaseFee(ctx, ts) diff --git a/lens/lily/struct.go b/lens/lily/struct.go index 64f497118..1f54053af 100644 --- a/lens/lily/struct.go +++ b/lens/lily/struct.go @@ -27,10 +27,12 @@ type LilyAPIStruct struct { Internal struct { Store func() adt.Store `perm:"read"` - LilyIndex func(ctx context.Context, config *LilyIndexConfig) (interface{}, error) `perm:"read"` - LilyWatch func(context.Context, *LilyWatchConfig) (*schedule.JobSubmitResult, error) `perm:"read"` - LilyWalk func(context.Context, *LilyWalkConfig) (*schedule.JobSubmitResult, error) `perm:"read"` - LilySurvey func(context.Context, *LilySurveyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilyIndex func(ctx context.Context, config *LilyIndexConfig) (interface{}, error) `perm:"read"` + LilyIndexIPLD func(ctx context.Context, config *LilyIndexIPLDConfig) (bool, error) `perm:"read"` + LilyIndexIPLDFile func(ctx context.Context, config *LilyIndexIPLDFileConfig) (bool, error) `perm:"read"` + LilyWatch func(context.Context, *LilyWatchConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilyWalk func(context.Context, *LilyWalkConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilySurvey func(context.Context, *LilySurveyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` LilyIndexNotify func(ctx context.Context, config *LilyIndexNotifyConfig) (interface{}, error) `perm:"read"` LilyWatchNotify func(ctx context.Context, config *LilyWatchNotifyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` @@ -81,6 +83,14 @@ type LilyAPIStruct struct { } } +func (s *LilyAPIStruct) LilyIndexIPLDFile(ctx context.Context, cfg *LilyIndexIPLDFileConfig) (bool, error) { + return s.Internal.LilyIndexIPLDFile(ctx, cfg) +} + +func (s *LilyAPIStruct) LilyIndexIPLD(ctx context.Context, cfg *LilyIndexIPLDConfig) (bool, error) { + return s.Internal.LilyIndexIPLD(ctx, cfg) +} + func (s *LilyAPIStruct) StateCompute(ctx context.Context, tsk types.TipSetKey) (interface{}, error) { return s.Internal.StateCompute(ctx, tsk) } diff --git a/model/actors/market/dealproposal.go b/model/actors/market/dealproposal.go index 829da0b0b..c1d025b5f 100644 --- a/model/actors/market/dealproposal.go +++ b/model/actors/market/dealproposal.go @@ -37,7 +37,7 @@ type MarketDealProposal struct { ProviderID string `pg:",notnull"` // The amount of FIL (in attoFIL) the client has pledged as collateral. ClientCollateral string `pg:",notnull"` - // The amount of FIL (in attoFIL) the provider has pledged as collateral. The Provider deal collateral is only slashed when a sector is terminated before the deal expires. + // The amount of FIL (in attoFIL) the provider has pledged as collateral. The Client deal collateral is only slashed when a sector is terminated before the deal expires. ProviderCollateral string `pg:",notnull"` // The amount of FIL (in attoFIL) that will be transferred from the client to the provider every epoch this deal is active for. StoragePricePerEpoch string `pg:",notnull"` diff --git a/pkg/core/array.go b/pkg/core/array.go new file mode 100644 index 000000000..25bdf9f72 --- /dev/null +++ b/pkg/core/array.go @@ -0,0 +1,135 @@ +package core + +import ( + "context" + + "github.com/filecoin-project/go-amt-ipld/v4" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/adt/diff" +) + +var log = logging.Logger("lily/core") + +type ArrayModification struct { + Key uint64 + Type ChangeType + Previous *typegen.Deferred + Current *typegen.Deferred +} + +type ArrayModifications []*ArrayModification + +func DiffArray(ctx context.Context, store adt.Store, child, parent adt.Array, childBw, parentBw int) (ArrayModifications, error) { + if childBw != parentBw { + diffContainer := &GenericArrayDiff{ + Added: []*ArrayModification{}, + Modified: []*ArrayModification{}, + Removed: []*ArrayModification{}, + } + log.Warn("diffing array using slow comparison") + if err := diff.CompareArray(child, parent, diffContainer); err != nil { + return nil, err + } + return diffContainer.AsArrayModifications() + } + changes, err := diff.Amt(ctx, parent, child, store, store, amt.UseTreeBitWidth(uint(childBw))) + if err != nil { + return nil, err + } + out := make(ArrayModifications, len(changes)) + for i, change := range changes { + out[i] = &ArrayModification{ + Key: change.Key, + Type: amtChangeTypeToGeneric(change.Type), + Previous: change.Before, + Current: change.After, + } + } + return out, nil +} + +func amtChangeTypeToGeneric(c amt.ChangeType) ChangeType { + switch c { + case amt.Add: + return ChangeTypeAdd + case amt.Remove: + return ChangeTypeRemove + case amt.Modify: + return ChangeTypeModify + } + panic("developer error") +} + +type GenericArrayDiff struct { + Added ArrayModifications + Modified ArrayModifications + Removed ArrayModifications +} + +func (t *GenericArrayDiff) AsArrayModifications() (ArrayModifications, error) { + out := make(ArrayModifications, len(t.Added)+len(t.Removed)+len(t.Modified)) + idx := 0 + for _, change := range t.Added { + out[idx] = &ArrayModification{ + Key: change.Key, + Type: ChangeTypeAdd, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + for _, change := range t.Removed { + out[idx] = &ArrayModification{ + Key: change.Key, + Type: ChangeTypeRemove, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + for _, change := range t.Modified { + out[idx] = &ArrayModification{ + Key: change.Key, + Type: ChangeTypeModify, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + return out, nil +} + +var _ diff.ArrayDiffer = &GenericArrayDiff{} + +func (t *GenericArrayDiff) Add(key uint64, val *typegen.Deferred) error { + t.Added = append(t.Added, &ArrayModification{ + Key: key, + Type: ChangeTypeAdd, + Previous: nil, + Current: val, + }) + return nil +} + +func (t *GenericArrayDiff) Modify(key uint64, from, to *typegen.Deferred) error { + t.Modified = append(t.Added, &ArrayModification{ + Key: key, + Type: ChangeTypeModify, + Previous: from, + Current: to, + }) + return nil +} + +func (t *GenericArrayDiff) Remove(key uint64, val *typegen.Deferred) error { + t.Removed = append(t.Added, &ArrayModification{ + Key: key, + Type: ChangeTypeRemove, + Previous: val, + Current: nil, + }) + return nil +} diff --git a/pkg/core/map.go b/pkg/core/map.go new file mode 100644 index 000000000..0f94dcf56 --- /dev/null +++ b/pkg/core/map.go @@ -0,0 +1,144 @@ +package core + +import ( + "context" + + "github.com/filecoin-project/go-hamt-ipld/v3" + "github.com/filecoin-project/go-state-types/abi" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/adt/diff" +) + +type MapModification struct { + Key []byte + Type ChangeType + Previous *typegen.Deferred + Current *typegen.Deferred +} + +type MapModifications []*MapModification + +func DiffMap(ctx context.Context, store adt.Store, child, parent adt.Map, childOpts, parentOpts *adt.MapOpts) (MapModifications, error) { + if !childOpts.Equal(parentOpts) { + diffContainer := &GenericMapDiff{ + Added: []*MapModification{}, + Modified: []*MapModification{}, + Removed: []*MapModification{}, + } + log.Warn("diffing array using slow comparison") + if err := diff.CompareMap(child, parent, diffContainer); err != nil { + return nil, err + } + return diffContainer.AsMapModifications() + } + changes, err := diff.Hamt(ctx, parent, child, store, store, hamt.UseHashFunction(hamt.HashFunc(childOpts.HashFunc)), hamt.UseTreeBitWidth(childOpts.Bitwidth)) + if err != nil { + return nil, err + } + out := make(MapModifications, len(changes)) + for i, change := range changes { + out[i] = &MapModification{ + Key: []byte(change.Key), + Type: hamtChangeTypeToGeneric(change.Type), + Previous: change.Before, + Current: change.After, + } + } + return out, nil +} + +func hamtChangeTypeToGeneric(c hamt.ChangeType) ChangeType { + switch c { + case hamt.Add: + return ChangeTypeAdd + case hamt.Remove: + return ChangeTypeRemove + case hamt.Modify: + return ChangeTypeModify + } + panic("developer error") +} + +type GenericMapDiff struct { + Added MapModifications + Modified MapModifications + Removed MapModifications +} + +func (t *GenericMapDiff) AsMapModifications() (MapModifications, error) { + out := make(MapModifications, len(t.Added)+len(t.Removed)+len(t.Modified)) + idx := 0 + for _, change := range t.Added { + out[idx] = &MapModification{ + Key: change.Key, + Type: ChangeTypeAdd, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + for _, change := range t.Removed { + out[idx] = &MapModification{ + Key: change.Key, + Type: ChangeTypeRemove, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + for _, change := range t.Modified { + out[idx] = &MapModification{ + Key: change.Key, + Type: ChangeTypeModify, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + return out, nil +} + +var _ diff.MapDiffer = &GenericMapDiff{} + +// An adt.Map key that just preserves the underlying string. +type StringKey string + +func (k StringKey) Key() string { + return string(k) +} + +func (t *GenericMapDiff) AsKey(key string) (abi.Keyer, error) { + return StringKey(key), nil +} + +func (t *GenericMapDiff) Add(key string, val *typegen.Deferred) error { + t.Added = append(t.Added, &MapModification{ + Key: []byte(key), + Type: ChangeTypeAdd, + Previous: nil, + Current: val, + }) + return nil +} + +func (t *GenericMapDiff) Modify(key string, from, to *typegen.Deferred) error { + t.Modified = append(t.Added, &MapModification{ + Key: []byte(key), + Type: ChangeTypeModify, + Previous: from, + Current: to, + }) + return nil +} + +func (t *GenericMapDiff) Remove(key string, val *typegen.Deferred) error { + t.Removed = append(t.Added, &MapModification{ + Key: []byte(key), + Type: ChangeTypeRemove, + Previous: val, + Current: nil, + }) + return nil +} diff --git a/pkg/core/reader.go b/pkg/core/reader.go new file mode 100644 index 000000000..19a0f6660 --- /dev/null +++ b/pkg/core/reader.go @@ -0,0 +1,40 @@ +package core + +import ( + "bytes" + "context" + "fmt" + "reflect" + + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" +) + +func StateReader(ctx context.Context, store adt.Store, c cid.Cid, fn interface{}) error { + raw := new(typegen.Deferred) + if err := store.Get(ctx, c, raw); err != nil { + return err + } + return StateReadDeferred(ctx, raw, fn) +} + +func StateReadDeferred(ctx context.Context, raw *typegen.Deferred, fn interface{}) error { + fnArg := reflect.TypeOf(fn).In(0) + if fnArg.Implements(reflect.TypeOf((*typegen.CBORUnmarshaler)(nil)).Elem()) { + p := reflect.New(fnArg.Elem()).Interface().(typegen.CBORUnmarshaler) + if err := p.UnmarshalCBOR(bytes.NewReader(raw.Raw)); err != nil { + return err + } + results := reflect.ValueOf(fn).Call([]reflect.Value{reflect.ValueOf(p)}) + if results[0].IsNil() { + return nil + } else { + return fmt.Errorf("error: %s", results[0]) + } + } + panic("here") + return nil + +} diff --git a/pkg/core/types.go b/pkg/core/types.go new file mode 100644 index 000000000..ca16b4bb3 --- /dev/null +++ b/pkg/core/types.go @@ -0,0 +1,25 @@ +package core + +// ChangeType denotes type of state change +type ChangeType uint8 + +const ( + ChangeTypeUnknown ChangeType = iota + ChangeTypeAdd + ChangeTypeRemove + ChangeTypeModify +) + +func (c ChangeType) String() string { + switch c { + case ChangeTypeUnknown: + return "unknown" + case ChangeTypeAdd: + return "add" + case ChangeTypeRemove: + return "remove" + case ChangeTypeModify: + return "modify" + } + panic("unreachable") +} diff --git a/pkg/core/versions.go b/pkg/core/versions.go new file mode 100644 index 000000000..94d853daa --- /dev/null +++ b/pkg/core/versions.go @@ -0,0 +1,69 @@ +package core + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + actorstypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +func ActorVersionForTipSet(ctx context.Context, ts *types.TipSet, ntwkVersionGetter func(ctx context.Context, epoch abi.ChainEpoch) network.Version) (actorstypes.Version, error) { + ntwkVersion := ntwkVersionGetter(ctx, ts.Height()) + return actorstypes.VersionForNetwork(ntwkVersion) +} + +var ( + AccountCodes = cid.NewSet() + CronCodes = cid.NewSet() + DataCapCodes = cid.NewSet() + InitCodes = cid.NewSet() + MarketCodes = cid.NewSet() + MinerCodes = cid.NewSet() + MultisigCodes = cid.NewSet() + PaychCodes = cid.NewSet() + PowerCodes = cid.NewSet() + RewardCodes = cid.NewSet() + SystemCodes = cid.NewSet() + VerifregCodes = cid.NewSet() +) + +func init() { + for _, a := range []string{actors.AccountKey, actors.CronKey, actors.DatacapKey, actors.InitKey, actors.MarketKey, actors.MinerKey, actors.MultisigKey, actors.PaychKey, actors.PowerKey, actors.RewardKey, actors.SystemKey, actors.VerifregKey} { + for _, v := range []int{0, 2, 3, 4, 5, 6, 7, 8, 9} { + code, ok := actors.GetActorCodeID(actorstypes.Version(v), a) + if !ok { + continue + } + switch a { + case actors.AccountKey: + AccountCodes.Add(code) + case actors.CronKey: + CronCodes.Add(code) + case actors.DatacapKey: + DataCapCodes.Add(code) + case actors.InitKey: + InitCodes.Add(code) + case actors.MarketKey: + MarketCodes.Add(code) + case actors.MinerKey: + MinerCodes.Add(code) + case actors.MultisigKey: + MultisigCodes.Add(code) + case actors.PaychKey: + PaychCodes.Add(code) + case actors.PowerKey: + PowerCodes.Add(code) + case actors.RewardKey: + RewardCodes.Add(code) + case actors.SystemKey: + SystemCodes.Add(code) + case actors.VerifregKey: + VerifregCodes.Add(code) + } + } + } +} diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go new file mode 100644 index 000000000..feed962d8 --- /dev/null +++ b/pkg/extract/actors.go @@ -0,0 +1,350 @@ +package extract + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + + "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/types" + "github.com/gammazero/workerpool" + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff" + dcapDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v1" + "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" + initDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" + "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff" + marketDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + minerDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff" + powerDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" + "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" + verifDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + verifDiffV2 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" + "github.com/filecoin-project/lily/pkg/extract/statetree" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("lily/extract/processor") + +type ActorStateChanges struct { + Current *types.TipSet + Executed *types.TipSet + RawActors map[address.Address]actors.DiffResult + MinerActors map[address.Address]actors.DiffResult + DatacapActor actors.DiffResult + InitActor actors.DiffResult + MarketActor actors.DiffResult + PowerActor actors.DiffResult + VerifregActor actors.DiffResult +} + +func (a ActorStateChanges) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.Int64("current", int64(a.Current.Height())), + attribute.Int64("executed", int64(a.Executed.Height())), + attribute.Int("miner_changes", len(a.MinerActors)), + } +} + +func (a ActorStateChanges) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range a.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + +type StateDiffResult struct { + ActorDiff actors.DiffResult + Actor *actors.Change +} + +func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, workers int) (*ActorStateChanges, error) { + actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) + if err != nil { + return nil, err + } + + curNtwkVersion, err := api.NetworkVersion(ctx, current.Key()) + if err != nil { + return nil, err + } + exeNtwkVersion, err := api.NetworkVersion(ctx, executed.Key()) + if err != nil { + return nil, err + } + curActorVersion, err := actortypes.VersionForNetwork(network.Version(curNtwkVersion)) + if err != nil { + return nil, err + } + exeActorVersion, err := actortypes.VersionForNetwork(network.Version(exeNtwkVersion)) + if err != nil { + return nil, err + } + log.Infow("extract actor states", + "current_network_version", curNtwkVersion, + "current_actor_version", curActorVersion, + "current_height", current.Height(), + "executed_network_version", exeNtwkVersion, + "executed_actor_version", exeActorVersion, + "executed_height", executed.Height(), + "workers", workers) + + var ( + // pool is the worker pool used to manage workers + pool = workerpool.New(workers) + // wg is the wait-group used to synchronize workers + wg = sync.WaitGroup{} + // resCh is the channel actor diff workers return results on + resCh = make(chan *StateDiffResult, workers*2) + // errCh is the channel actor diff workers return errors on + errCh = make(chan error, workers*2) + // done is the channel used for signaling workers have stopped execution + done = make(chan struct{}) + // cancel is the channel used to cancel any active workers + cancel = make(chan struct{}) + // scheduledWorkers is the number of workers scheduled for execution, its value decreases as workers complete + scheduledWorkers = int64(0) + // workerCtx is the context used by the workers. + // TODO a method deep in the call stack of this function is not respecting context cancellation + workerCtx = context.TODO() + ) + + for addr, change := range actorChanges { + act := &actors.Change{ + Address: addr, + Executed: change.Executed, + ExeVersion: exeActorVersion, + Current: change.Current, + CurVersion: curActorVersion, + Type: change.ChangeType, + } + + // submit two workers, one to extract raw actor states and one to diff individual actor states + wg.Add(1) + scheduledWorkers++ + pool.Submit(func() { + defer func() { + wg.Done() + atomic.AddInt64(&scheduledWorkers, -1) + + }() + res, err := diffRawActorState(workerCtx, api, act) + if err != nil { + // attempt to send the error or bail if canceled + for { + select { + case <-cancel: + return + case errCh <- err: + return + } + } + } + // attempt to send the result or bail if canceled + for { + select { + case <-cancel: + return + case resCh <- res: + return + } + } + }) + + wg.Add(1) + scheduledWorkers++ + pool.Submit(func() { + defer func() { + wg.Done() + atomic.AddInt64(&scheduledWorkers, -1) + + }() + res, ok, err := diffTypedActorState(workerCtx, api, act) + if err != nil { + // attempt to send the error or bail if canceled + for { + select { + case <-cancel: + return + case errCh <- err: + return + } + } + } + if ok { + // attempt to send the result or bail if canceled + for { + select { + case <-cancel: + return + case resCh <- res: + return + } + } + } + // Not all actors have their state diffed, for example account actors have no state to diff: their state is empty. + log.Debugw("no actor diff for actor", "current_network", curNtwkVersion, "executed_network", exeNtwkVersion, + "address", act.Address, "current_code", act.Current.Code, "current_version", + act.CurVersion, "executed_code", act.Executed.Code, "executed_version", act.ExeVersion) + }) + } + + // wait for workers to complete then signal work is done. + go func() { + wg.Wait() + done <- struct{}{} + log.Debugw("cleaned up happy path go routine") + }() + + // stop the worker pool, dropping any scheduled workers, and complete any wait-groups for the case a workers were canceled. + cleanup := func() { + log.Debug("cancel any remaining workers") + close(cancel) + pool.Stop() + log.Debug("worker pool stopped") + for i := scheduledWorkers; i > 0; i-- { + wg.Done() + } + close(resCh) + close(errCh) + } + + // result of diffing all actor states + out := &ActorStateChanges{ + Current: current, + Executed: executed, + MinerActors: make(map[address.Address]actors.DiffResult, len(actorChanges)), // there are at most actorChanges entries + RawActors: make(map[address.Address]actors.DiffResult, len(actorChanges)), // there are at most actorChanges entries + } + for { + log.Debugw("diff jobs todo %d", scheduledWorkers) + select { + // canceling the context or receiving an error causes all workers to stop and drops any scheduled workers. + case <-workerCtx.Done(): + log.Errorw("context canceled", "error", workerCtx.Err()) + cleanup() + <-done + return nil, workerCtx.Err() + case err := <-errCh: + log.Errorw("worker received error while processing", "error", err) + cleanup() + <-done + return nil, err + + // happy path, all workers completed and we can return with a result + case <-done: + log.Debug("done processing") + cleanup() + return out, nil + + // result returned from a worker corresponding to the actor it diffed. + case res := <-resCh: + switch v := res.ActorDiff.(type) { + case *rawdiff.StateDiffResult: + out.RawActors[res.Actor.Address] = v + case *minerDiffV1.StateDiffResult: + out.MinerActors[res.Actor.Address] = v + case *initDiffV1.StateDiffResult: + out.InitActor = v + case *powerDiffV1.StateDiffResult: + out.PowerActor = v + case *marketDiffV1.StateDiffResult: + out.MarketActor = v + case *dcapDiffV1.StateDiffResult: + out.DatacapActor = v + case *verifDiffV1.StateDiffResult, *verifDiffV2.StateDiffResult: + out.VerifregActor = v + default: + // this indicates a developer error so it's okay to panic, the code is wrong and must be fixed. + panic(fmt.Errorf("unknown StateDiffResult type: %T", v)) + } + } + } + +} + +// diffRawActorState is called on every actor with a state change and returns their state. +func diffRawActorState(ctx context.Context, api tasks.DataSource, act *actors.Change) (*StateDiffResult, error) { + methods, handler, err := rawdiff.StateDiffFor(act.CurVersion) + if err != nil { + return nil, err + } + + actorDiff := &actors.StateDiffer{ + Methods: methods, + ActorHandler: handler, + ReportHandler: func(reports []actors.DifferReport) error { + for _, report := range reports { + log.Debugw("reporting", "type", report.DiffType, "duration", report.Duration) + } + return nil + }, + } + res, err := actorDiff.ActorDiff(ctx, api, act) + if err != nil { + return nil, err + } + return &StateDiffResult{ + ActorDiff: res, + Actor: act, + }, nil +} + +// diffTypedActorState is called on every actor with a state change and returns artifacts from diffing their internal state +// such as miner hamts, market amts, etc. It returns false if the actor `act` doesn't have a corresponding state diff implementation. +func diffTypedActorState(ctx context.Context, api tasks.DataSource, act *actors.Change) (*StateDiffResult, bool, error) { + var ( + methods []actors.ActorDiffMethods + handler actors.ActorHandlerFn + err error + ) + if core.DataCapCodes.Has(act.Current.Code) { + methods, handler, err = datacapdiff.StateDiffFor(act.CurVersion) + } else if core.MinerCodes.Has(act.Current.Code) { + methods, handler, err = minerdiff.StateDiffFor(act.CurVersion) + } else if core.InitCodes.Has(act.Current.Code) { + methods, handler, err = initdiff.StateDiffFor(act.CurVersion) + } else if core.PowerCodes.Has(act.Current.Code) { + methods, handler, err = powerdiff.StateDiffFor(act.CurVersion) + } else if core.MarketCodes.Has(act.Current.Code) { + methods, handler, err = marketdiff.StateDiffFor(act.CurVersion) + } else if core.VerifregCodes.Has(act.Current.Code) { + methods, handler, err = verifregdiff.StateDiffFor(act.CurVersion) + } else { + return nil, false, nil + } + + if err != nil { + return nil, false, nil + } + + actorDiff := &actors.StateDiffer{ + Methods: methods, + ActorHandler: handler, + ReportHandler: func(reports []actors.DifferReport) error { + for _, report := range reports { + log.Debugw("reporting", "type", report.DiffType, "duration", report.Duration) + } + return nil + }, + } + res, err := actorDiff.ActorDiff(ctx, api, act) + if err != nil { + return nil, false, err + } + return &StateDiffResult{ + ActorDiff: res, + Actor: act, + }, true, nil +} diff --git a/pkg/extract/actors/datacapdiff/v1/allowances.go b/pkg/extract/actors/datacapdiff/v1/allowances.go new file mode 100644 index 000000000..36edf6fb6 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/v1/allowances.go @@ -0,0 +1,129 @@ +package v1 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/datacap" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +type AllowanceChange struct { + Owner []byte `cborgen:"owner"` + Operator []byte `cborgen:"operator"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +type AllowanceChangeMap map[address.Address][]*AllowanceChange + +const KindDataCapAllowance = "datacap_allowance" + +func (a AllowanceChangeMap) Kind() actors.ActorStateKind { + return KindDataCapAllowance +} + +func (a AllowanceChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + topNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for owner, changes := range a { + innerNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, change := range changes { + if err := innerNode.Put(core.StringKey(change.Operator), change); err != nil { + return cid.Undef, err + } + } + innerRoot, err := innerNode.Root() + if err != nil { + return cid.Undef, err + } + if err := topNode.Put(abi.IdAddrKey(owner), typegen.CborCid(innerRoot)); err != nil { + return cid.Undef, err + } + } + return topNode.Root() +} + +type Allowance struct{} + +func (a Allowance) Type() string { + return KindDataCapAllowance +} + +func (Allowance) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindDataCapAllowance, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffAllowances(ctx, api, act) +} + +func DiffAllowances(ctx context.Context, api tasks.DataSource, act *actors.Change) (AllowanceChangeMap, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, DatacapAllowancesMapLoader) + if err != nil { + return nil, err + } + out := make(AllowanceChangeMap) + for _, change := range mapChange { + ownerId, err := abi.ParseUIntKey(string(change.Key)) + if err != nil { + return nil, err + } + ownerAddress, err := address.NewIDAddress(ownerId) + if err != nil { + return nil, err + } + ownerMapChanges, err := diffOwnerMap(ctx, api, act, ownerAddress, change.Key) + if err != nil { + return nil, err + } + out[ownerAddress] = ownerMapChanges + } + return out, nil +} + +func diffOwnerMap(ctx context.Context, api tasks.DataSource, act *actors.Change, ownerAddress address.Address, ownerKey []byte) ([]*AllowanceChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + datacapState := i.(datacap.State) + clientAllocationMap, err := datacapState.AllowanceMapForOwner(ownerAddress) + if err != nil { + return nil, nil, err + } + return clientAllocationMap, &adt.MapOpts{ + Bitwidth: datacapState.AllowanceMapBitWidth(), + HashFunc: datacapState.AllowanceMapHashFunction(), + }, nil + }) + if err != nil { + return nil, err + } + out := make([]*AllowanceChange, 0, len(mapChange)) + for _, change := range mapChange { + out = append(out, &AllowanceChange{ + Owner: ownerKey, + Operator: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + }) + } + return out, nil +} diff --git a/pkg/extract/actors/datacapdiff/v1/balances.go b/pkg/extract/actors/datacapdiff/v1/balances.go new file mode 100644 index 000000000..eca1116a6 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/v1/balances.go @@ -0,0 +1,80 @@ +package v1 + +import ( + "context" + "time" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("lily/extract/actors/balance/v9") + +type BalanceChange struct { + Client []byte `cborgen:"client"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +type BalanceChangeList []*BalanceChange + +const KindDataCapBalance = "datacap_balance" + +func (b BalanceChangeList) Kind() actors.ActorStateKind { + return KindDataCapBalance +} + +func (b BalanceChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range b { + if err := node.Put(core.StringKey(l.Client), l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type Balance struct{} + +func (b Balance) Type() string { + return KindDataCapBalance +} + +func (Balance) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindDataCapBalance, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffBalances(ctx, api, act) +} + +func DiffBalances(ctx context.Context, api tasks.DataSource, act *actors.Change) (BalanceChangeList, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, DatacapBalancesMapLoader) + if err != nil { + return nil, err + } + + out := make(BalanceChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &BalanceChange{ + Client: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/datacapdiff/v1/cbor_gen.go b/pkg/extract/actors/datacapdiff/v1/cbor_gen.go new file mode 100644 index 000000000..af7fa4862 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/v1/cbor_gen.go @@ -0,0 +1,601 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v1 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *AllowanceChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Owner ([]uint8) (slice) + if len("owner") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"owner\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("owner"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("owner")); err != nil { + return err + } + + if len(t.Owner) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Owner was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Owner))); err != nil { + return err + } + + if _, err := cw.Write(t.Owner[:]); err != nil { + return err + } + + // t.Operator ([]uint8) (slice) + if len("operator") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"operator\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("operator"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("operator")); err != nil { + return err + } + + if len(t.Operator) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Operator was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Operator))); err != nil { + return err + } + + if _, err := cw.Write(t.Operator[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *AllowanceChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = AllowanceChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("AllowanceChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Owner ([]uint8) (slice) + case "owner": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Owner: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Owner = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Owner[:]); err != nil { + return err + } + // t.Operator ([]uint8) (slice) + case "operator": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Operator: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Operator = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Operator[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *BalanceChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Client ([]uint8) (slice) + if len("client") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"client\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("client"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("client")); err != nil { + return err + } + + if len(t.Client) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Client was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Client))); err != nil { + return err + } + + if _, err := cw.Write(t.Client[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *BalanceChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = BalanceChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("BalanceChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Client ([]uint8) (slice) + case "client": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Client: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Client = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Client[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Balances (cid.Cid) (struct) + if len("balances") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"balances\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("balances"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("balances")); err != nil { + return err + } + + if t.Balances == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Balances); err != nil { + return xerrors.Errorf("failed to write cid field t.Balances: %w", err) + } + } + + // t.Allowances (cid.Cid) (struct) + if len("allowances") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"allowances\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allowances"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("allowances")); err != nil { + return err + } + + if t.Allowances == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Allowances); err != nil { + return xerrors.Errorf("failed to write cid field t.Allowances: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Balances (cid.Cid) (struct) + case "balances": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Balances: %w", err) + } + + t.Balances = &c + } + + } + // t.Allowances (cid.Cid) (struct) + case "allowances": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Allowances: %w", err) + } + + t.Allowances = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/datacapdiff/v1/load.go b/pkg/extract/actors/datacapdiff/v1/load.go new file mode 100644 index 000000000..681c646a1 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/v1/load.go @@ -0,0 +1,37 @@ +package v1 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/datacap" +) + +var DatacapStateLoader = func(s adt.Store, act *types.Actor) (interface{}, error) { + return datacap.Load(s, act) +} + +var DatacapBalancesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + datacapState := m.(datacap.State) + balanceMap, err := datacapState.VerifiedClients() + if err != nil { + return nil, nil, err + } + return balanceMap, &adt.MapOpts{ + Bitwidth: datacapState.VerifiedClientsMapBitWidth(), + HashFunc: datacapState.VerifiedClientsMapHashFunction(), + }, nil +} + +var DatacapAllowancesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + datacapState := m.(datacap.State) + allowanceMap, err := datacapState.AllowanceMap() + if err != nil { + return nil, nil, err + } + return allowanceMap, &adt.MapOpts{ + Bitwidth: datacapState.AllowanceMapBitWidth(), + HashFunc: datacapState.VerifiedClientsMapHashFunction(), + }, nil + +} diff --git a/pkg/extract/actors/datacapdiff/v1/state.go b/pkg/extract/actors/datacapdiff/v1/state.go new file mode 100644 index 000000000..52d0f8423 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/v1/state.go @@ -0,0 +1,57 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case AllowanceChangeMap: + stateDiff.AllowanceChanges = v + case BalanceChangeList: + stateDiff.BalanceChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) + } + } + return stateDiff, nil +} + +type StateDiffResult struct { + BalanceChanges BalanceChangeList + AllowanceChanges AllowanceChangeMap +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { + out := &StateChange{} + + if balances := sd.BalanceChanges; balances != nil { + root, err := balances.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Balances = &root + } + if allowances := sd.AllowanceChanges; allowances != nil { + root, err := allowances.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Allowances = &root + } + return out, nil +} + +type StateChange struct { + Balances *cid.Cid `cborgen:"balances"` + Allowances *cid.Cid `cborgen:"allowances"` +} diff --git a/pkg/extract/actors/datacapdiff/version.go b/pkg/extract/actors/datacapdiff/version.go new file mode 100644 index 000000000..d8640058e --- /dev/null +++ b/pkg/extract/actors/datacapdiff/version.go @@ -0,0 +1,18 @@ +package datacapdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v1" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + switch av { + case actortypes.Version9: + return []actors.ActorDiffMethods{v1.Allowance{}, v1.Balance{}}, v1.ActorStateChangeHandler, nil + } + return nil, nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go new file mode 100644 index 000000000..9ceb58c53 --- /dev/null +++ b/pkg/extract/actors/executor.go @@ -0,0 +1,57 @@ +package actors + +import ( + "context" + "time" + + logging "github.com/ipfs/go-log/v2" + + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("lily/extract/actors") + +type DifferReport struct { + DiffType string + StartTime time.Time + Duration time.Duration + Result ActorStateChange +} + +type StateDiffer struct { + Methods []ActorDiffMethods + ReportHandler ReportHandlerFn + ActorHandler ActorHandlerFn +} + +type ReportHandlerFn = func(reports []DifferReport) error +type ActorHandlerFn = func(changes []ActorStateChange) (DiffResult, error) + +func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act *Change) (DiffResult, error) { + out := make([]DifferReport, len(s.Methods)) + for i, fn := range s.Methods { + start := time.Now() + res, err := fn.Diff(ctx, api, act) + if err != nil { + return nil, err + } + out[i] = DifferReport{ + DiffType: fn.Type(), + StartTime: start, + Duration: time.Since(start), + Result: res, + } + } + + if s.ReportHandler != nil { + if err := s.ReportHandler(out); err != nil { + return nil, err + } + } + + var results []ActorStateChange + for _, report := range out { + results = append(results, report.Result) + } + return s.ActorHandler(results) +} diff --git a/pkg/extract/actors/generic/array.go b/pkg/extract/actors/generic/array.go new file mode 100644 index 000000000..ed22e70fe --- /dev/null +++ b/pkg/extract/actors/generic/array.go @@ -0,0 +1,80 @@ +package generic + +import ( + "context" + + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +func DiffActorArray(ctx context.Context, api tasks.DataSource, act *actors.Change, actorStateLoader ActorStateLoader, actorArrayLoader ActorStateArrayLoader) (core.ArrayModifications, error) { + if act.Type == core.ChangeTypeRemove { + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + executedArray, _, err := actorArrayLoader(executedActor) + if err != nil { + return nil, err + } + + var out core.ArrayModifications + var v typegen.Deferred + if err := executedArray.ForEach(&v, func(key int64) error { + value := v + out = append(out, &core.ArrayModification{ + Key: uint64(key), + Type: core.ChangeTypeRemove, + Previous: &value, + Current: nil, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil + } + + currentActor, err := actorStateLoader(api.Store(), act.Current) + if err != nil { + return nil, err + } + + currentArray, currentArrayBitWidth, err := actorArrayLoader(currentActor) + if err != nil { + return nil, err + } + if act.Type == core.ChangeTypeAdd { + var out core.ArrayModifications + var v typegen.Deferred + if err := currentArray.ForEach(&v, func(key int64) error { + value := v + out = append(out, &core.ArrayModification{ + Key: uint64(key), + Type: core.ChangeTypeAdd, + Previous: nil, + Current: &value, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil + } + + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + executedArray, executedArrayBitWidth, err := actorArrayLoader(executedActor) + if err != nil { + return nil, err + } + + return core.DiffArray(ctx, api.Store(), currentArray, executedArray, currentArrayBitWidth, executedArrayBitWidth) +} diff --git a/pkg/extract/actors/generic/map.go b/pkg/extract/actors/generic/map.go new file mode 100644 index 000000000..14d32f96b --- /dev/null +++ b/pkg/extract/actors/generic/map.go @@ -0,0 +1,82 @@ +package generic + +import ( + "context" + + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +func DiffActorMap(ctx context.Context, api tasks.DataSource, act *actors.Change, actorStateLoader ActorStateLoader, actorMapLoader ActorStateMapLoader) (core.MapModifications, error) { + if act.Type == core.ChangeTypeRemove { + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + executedMap, _, err := actorMapLoader(executedActor) + if err != nil { + return nil, err + } + + var out core.MapModifications + var v typegen.Deferred + if err := executedMap.ForEach(&v, func(key string) error { + value := v + out = append(out, &core.MapModification{ + Key: []byte(key), + Type: core.ChangeTypeRemove, + Previous: &value, + Current: nil, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil + } + + currentActor, err := actorStateLoader(api.Store(), act.Current) + if err != nil { + return nil, err + } + + currentMap, currentMapOpts, err := actorMapLoader(currentActor) + if err != nil { + return nil, err + } + + if act.Type == core.ChangeTypeAdd { + var out core.MapModifications + var v typegen.Deferred + if err := currentMap.ForEach(&v, func(key string) error { + value := v + out = append(out, &core.MapModification{ + Key: []byte(key), + Type: core.ChangeTypeAdd, + Previous: nil, + Current: &value, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil + + } + + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + executedMap, executedMapOpts, err := actorMapLoader(executedActor) + if err != nil { + return nil, err + } + + return core.DiffMap(ctx, api.Store(), currentMap, executedMap, currentMapOpts, executedMapOpts) +} diff --git a/pkg/extract/actors/generic/types.go b/pkg/extract/actors/generic/types.go new file mode 100644 index 000000000..8d44e13de --- /dev/null +++ b/pkg/extract/actors/generic/types.go @@ -0,0 +1,16 @@ +package generic + +import ( + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + + "github.com/filecoin-project/lily/chain/actors/adt" +) + +var log = logging.Logger("lily/extract/diff") + +// TODO use go-state-types store + +type ActorStateArrayLoader = func(interface{}) (adt.Array, int, error) +type ActorStateLoader = func(adt.Store, *types.Actor) (interface{}, error) +type ActorStateMapLoader = func(interface{}) (adt.Map, *adt.MapOpts, error) diff --git a/pkg/extract/actors/initdiff/v1/addresses.go b/pkg/extract/actors/initdiff/v1/addresses.go new file mode 100644 index 000000000..b99f3c7ed --- /dev/null +++ b/pkg/extract/actors/initdiff/v1/addresses.go @@ -0,0 +1,81 @@ +package v1 + +import ( + "context" + "time" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/init") + +var _ actors.ActorStateChange = (*AddressChangeList)(nil) + +type AddressChange struct { + Address []byte `cborgen:"address"` + Current *typegen.Deferred `cborgen:"current_actorID"` + Previous *typegen.Deferred `cborgen:"previous_actorID"` + Change core.ChangeType `cborgen:"change"` +} + +type AddressChangeList []*AddressChange + +const KindInitAddresses = "init_addresses" + +func (a AddressChangeList) Kind() actors.ActorStateKind { + return KindInitAddresses +} + +func (a AddressChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range a { + if err := node.Put(core.StringKey(l.Address), l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type Addresses struct{} + +func (a Addresses) Type() string { + return KindInitAddresses +} + +func (Addresses) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindInitAddresses, zap.Inline(act), "duration", time.Since(start)) + }() + return AddressesDiff(ctx, api, act) +} + +func AddressesDiff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, InitStateLoader, InitAddressesMapLoader) + if err != nil { + return nil, err + } + out := make(AddressChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &AddressChange{ + Address: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/initdiff/v1/cbor_gen.go b/pkg/extract/actors/initdiff/v1/cbor_gen.go new file mode 100644 index 000000000..e5c286636 --- /dev/null +++ b/pkg/extract/actors/initdiff/v1/cbor_gen.go @@ -0,0 +1,319 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v1 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *AddressChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Address ([]uint8) (slice) + if len("address") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"address\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("address"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("address")); err != nil { + return err + } + + if len(t.Address) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Address was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Address))); err != nil { + return err + } + + if _, err := cw.Write(t.Address[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current_actorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_actorID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_actorID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current_actorID")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous_actorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_actorID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_actorID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous_actorID")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *AddressChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = AddressChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("AddressChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Address ([]uint8) (slice) + case "address": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Address: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Address = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Address[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current_actorID": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous_actorID": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Addresses (cid.Cid) (struct) + if len("addresses") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"addresses\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("addresses"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("addresses")); err != nil { + return err + } + + if t.Addresses == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Addresses); err != nil { + return xerrors.Errorf("failed to write cid field t.Addresses: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Addresses (cid.Cid) (struct) + case "addresses": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Addresses: %w", err) + } + + t.Addresses = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/initdiff/v1/load.go b/pkg/extract/actors/initdiff/v1/load.go new file mode 100644 index 000000000..22d3edd85 --- /dev/null +++ b/pkg/extract/actors/initdiff/v1/load.go @@ -0,0 +1,24 @@ +package v1 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + init_ "github.com/filecoin-project/lily/chain/actors/builtin/init" +) + +var InitStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return init_.Load(store, act) +} + +var InitAddressesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + initState := m.(init_.State) + addressesMap, err := initState.AddressMap() + if err != nil { + return nil, nil, err + } + return addressesMap, &adt.MapOpts{ + Bitwidth: initState.AddressMapBitWidth(), + HashFunc: initState.AddressMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/initdiff/v1/state.go b/pkg/extract/actors/initdiff/v1/state.go new file mode 100644 index 000000000..72796789b --- /dev/null +++ b/pkg/extract/actors/initdiff/v1/state.go @@ -0,0 +1,73 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case AddressChangeList: + stateDiff.AddressesChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) + } + } + return stateDiff, nil +} + +type StateDiffResult struct { + AddressesChanges AddressChangeList +} + +func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { + out := &StateChange{} + + if addresses := sdr.AddressesChanges; addresses != nil { + root, err := addresses.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Addresses = &root + } + return out, nil +} + +type StateChange struct { + Addresses *cid.Cid `cborgen:"addresses"` +} + +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{AddressesChanges: AddressChangeList{}} + + if sc.Addresses != nil { + addressMap, err := adt.AsMap(s, *sc.Addresses, 5) + if err != nil { + return nil, err + } + + addresses := AddressChangeList{} + addressChange := new(AddressChange) + if err := addressMap.ForEach(addressChange, func(key string) error { + val := new(AddressChange) + *val = *addressChange + // NB: not required + // TODO consider removing they key from the structure. + val.Address = []byte(key) + addresses = append(addresses, val) + return nil + }); err != nil { + return nil, err + } + } + return out, nil +} diff --git a/pkg/extract/actors/initdiff/version.go b/pkg/extract/actors/initdiff/version.go new file mode 100644 index 000000000..c1183ed10 --- /dev/null +++ b/pkg/extract/actors/initdiff/version.go @@ -0,0 +1,20 @@ +package initdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + switch av { + // TODO less than eaual to version 9 + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return []actors.ActorDiffMethods{v1.Addresses{}}, v1.ActorStateChangeHandler, nil + } + return nil, nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/marketdiff/v1/cbor_gen.go b/pkg/extract/actors/marketdiff/v1/cbor_gen.go new file mode 100644 index 000000000..e578e4b0d --- /dev/null +++ b/pkg/extract/actors/marketdiff/v1/cbor_gen.go @@ -0,0 +1,525 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v1 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Deals (cid.Cid) (struct) + if len("deals") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"deals\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("deals"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("deals")); err != nil { + return err + } + + if t.Deals == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Deals); err != nil { + return xerrors.Errorf("failed to write cid field t.Deals: %w", err) + } + } + + // t.Proposals (cid.Cid) (struct) + if len("proposals") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"proposals\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("proposals"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("proposals")); err != nil { + return err + } + + if t.Proposals == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Proposals); err != nil { + return xerrors.Errorf("failed to write cid field t.Proposals: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Deals (cid.Cid) (struct) + case "deals": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Deals: %w", err) + } + + t.Deals = &c + } + + } + // t.Proposals (cid.Cid) (struct) + case "proposals": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Proposals: %w", err) + } + + t.Proposals = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ProposalChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.DealID (uint64) (uint64) + if len("dealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"dealID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("dealID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("dealID")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ProposalChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ProposalChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ProposalChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.DealID (uint64) (uint64) + case "dealID": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *DealChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.DealID (uint64) (uint64) + if len("dealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"dealID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("dealID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("dealID")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *DealChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = DealChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DealChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.DealID (uint64) (uint64) + case "dealID": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/marketdiff/v1/deals.go b/pkg/extract/actors/marketdiff/v1/deals.go new file mode 100644 index 000000000..933646aa8 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v1/deals.go @@ -0,0 +1,77 @@ +package v1 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +type DealChange struct { + DealID uint64 `cborgen:"dealID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +type DealChangeList []*DealChange + +const KindMarketDeal = "market_deal" + +func (p DealChangeList) Kind() actors.ActorStateKind { + return KindMarketDeal +} + +func (p DealChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(abi.UIntKey(l.DealID), l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type Deals struct{} + +func (d Deals) Type() string { + return KindMarketDeal +} + +func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMarketDeal, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffDeals(ctx, api, act) +} + +func DiffDeals(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + arrayChanges, err := generic.DiffActorArray(ctx, api, act, MarketStateLoader, MarketDealsArrayLoader) + if err != nil { + return nil, err + } + out := make(DealChangeList, len(arrayChanges)) + for i, change := range arrayChanges { + out[i] = &DealChange{ + DealID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/marketdiff/v1/load.go b/pkg/extract/actors/marketdiff/v1/load.go new file mode 100644 index 000000000..2395f3305 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v1/load.go @@ -0,0 +1,30 @@ +package v1 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/market" +) + +var MarketStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return market.Load(store, act) +} + +var MarketDealsArrayLoader = func(m interface{}) (adt.Array, int, error) { + marketState := m.(market.State) + dealsArray, err := marketState.States() + if err != nil { + return nil, -1, err + } + return dealsArray.AsArray(), marketState.DealStatesAmtBitwidth(), nil +} + +var MarketProposlasArrayLoader = func(m interface{}) (adt.Array, int, error) { + marketState := m.(market.State) + proposalsArray, err := marketState.Proposals() + if err != nil { + return nil, -1, err + } + return proposalsArray.AsArray(), marketState.DealProposalsAmtBitwidth(), nil +} diff --git a/pkg/extract/actors/marketdiff/v1/proposals.go b/pkg/extract/actors/marketdiff/v1/proposals.go new file mode 100644 index 000000000..c4d4ba52d --- /dev/null +++ b/pkg/extract/actors/marketdiff/v1/proposals.go @@ -0,0 +1,84 @@ +package v1 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/market") + +type ProposalChange struct { + DealID uint64 `cborgen:"dealID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *ProposalChange) Key() string { + return abi.UIntKey(t.DealID).Key() +} + +type ProposalChangeList []*ProposalChange + +const KindMarketProposal = "market_proposal" + +func (p ProposalChangeList) Kind() actors.ActorStateKind { + return KindMarketProposal +} + +func (p ProposalChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type Proposals struct{} + +func (p Proposals) Type() string { + return KindMarketProposal +} + +func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMarketProposal, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffProposals(ctx, api, act) +} + +func DiffProposals(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + arrayChanges, err := generic.DiffActorArray(ctx, api, act, MarketStateLoader, MarketProposlasArrayLoader) + if err != nil { + return nil, err + } + out := make(ProposalChangeList, len(arrayChanges)) + for i, change := range arrayChanges { + out[i] = &ProposalChange{ + DealID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/marketdiff/v1/state.go b/pkg/extract/actors/marketdiff/v1/state.go new file mode 100644 index 000000000..9f6899d83 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v1/state.go @@ -0,0 +1,124 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case DealChangeList: + stateDiff.DealStateChanges = v + case ProposalChangeList: + stateDiff.DealProposalChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) + } + } + return stateDiff, nil +} + +type StateDiffResult struct { + DealStateChanges DealChangeList + DealProposalChanges ProposalChangeList +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { + out := &StateChange{} + + if deals := sd.DealStateChanges; deals != nil { + root, err := deals.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Deals = &root + } + + if proposals := sd.DealProposalChanges; proposals != nil { + root, err := proposals.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Proposals = &root + } + return out, nil +} + +type StateChange struct { + Deals *cid.Cid `cborgen:"deals"` + Proposals *cid.Cid `cborgen:"proposals"` +} + +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + DealStateChanges: DealChangeList{}, + DealProposalChanges: ProposalChangeList{}, + } + + if sc.Deals != nil { + dealsMap, err := adt.AsMap(s, *sc.Deals, 5) + if err != nil { + return nil, err + } + deals := DealChangeList{} + dealChange := new(DealChange) + if err := dealsMap.ForEach(dealChange, func(key string) error { + val := new(DealChange) + *val = *dealChange + // NB: this is optinal since the dealChange already contains the dealID + // TODO consider removeing the key from the structure to save space + dealID, err := abi.ParseUIntKey(key) + if err != nil { + return err + } + if dealID != val.DealID { + panic("here") + } + val.DealID = dealID + deals = append(deals, val) + return nil + }); err != nil { + return nil, err + } + out.DealStateChanges = deals + } + + if sc.Proposals != nil { + propsMap, err := adt.AsMap(s, *sc.Proposals, 5) + if err != nil { + return nil, err + } + props := ProposalChangeList{} + propChange := new(ProposalChange) + if err := propsMap.ForEach(propChange, func(key string) error { + val := new(ProposalChange) + *val = *propChange + // NB: this is optinal since the dealChange already contains the dealID + // TODO consider removeing the key from the structure to save space + dealID, err := abi.ParseUIntKey(key) + if err != nil { + return err + } + if dealID != val.DealID { + panic("here") + } + val.DealID = dealID + props = append(props, val) + return nil + }); err != nil { + return nil, err + } + out.DealProposalChanges = props + } + return out, nil +} diff --git a/pkg/extract/actors/marketdiff/version.go b/pkg/extract/actors/marketdiff/version.go new file mode 100644 index 000000000..610e7bab5 --- /dev/null +++ b/pkg/extract/actors/marketdiff/version.go @@ -0,0 +1,22 @@ +package marketdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + switch av { + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return []actors.ActorDiffMethods{ + v1.Deals{}, + v1.Proposals{}, + }, v1.ActorStateChangeHandler, nil + } + return nil, nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/minerdiff/v1/cbor_gen.go b/pkg/extract/actors/minerdiff/v1/cbor_gen.go new file mode 100644 index 000000000..522da21fa --- /dev/null +++ b/pkg/extract/actors/minerdiff/v1/cbor_gen.go @@ -0,0 +1,913 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v1 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *SectorStatusChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Removed (bitfield.BitField) (struct) + if len("removed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"removed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("removed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("removed")); err != nil { + return err + } + + if err := t.Removed.MarshalCBOR(cw); err != nil { + return err + } + + // t.Recovering (bitfield.BitField) (struct) + if len("recovering") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"recovering\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("recovering"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("recovering")); err != nil { + return err + } + + if err := t.Recovering.MarshalCBOR(cw); err != nil { + return err + } + + // t.Faulted (bitfield.BitField) (struct) + if len("faulted") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"faulted\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("faulted"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("faulted")); err != nil { + return err + } + + if err := t.Faulted.MarshalCBOR(cw); err != nil { + return err + } + + // t.Recovered (bitfield.BitField) (struct) + if len("recovered") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"recovered\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("recovered"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("recovered")); err != nil { + return err + } + + if err := t.Recovered.MarshalCBOR(cw); err != nil { + return err + } + return nil +} + +func (t *SectorStatusChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = SectorStatusChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SectorStatusChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Removed (bitfield.BitField) (struct) + case "removed": + + { + + if err := t.Removed.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Removed: %w", err) + } + + } + // t.Recovering (bitfield.BitField) (struct) + case "recovering": + + { + + if err := t.Recovering.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Recovering: %w", err) + } + + } + // t.Faulted (bitfield.BitField) (struct) + case "faulted": + + { + + if err := t.Faulted.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Faulted: %w", err) + } + + } + // t.Recovered (bitfield.BitField) (struct) + case "recovered": + + { + + if err := t.Recovered.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Recovered: %w", err) + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *PreCommitChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.SectorNumber ([]uint8) (slice) + if len("sector_number") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_number\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_number"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_number")); err != nil { + return err + } + + if len(t.SectorNumber) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.SectorNumber was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.SectorNumber))); err != nil { + return err + } + + if _, err := cw.Write(t.SectorNumber[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current_pre_commit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_pre_commit\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_pre_commit"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current_pre_commit")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous_pre_commit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_pre_commit\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_pre_commit"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous_pre_commit")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *PreCommitChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = PreCommitChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PreCommitChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.SectorNumber ([]uint8) (slice) + case "sector_number": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.SectorNumber: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.SectorNumber = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.SectorNumber[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current_pre_commit": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous_pre_commit": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *SectorChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.SectorNumber (uint64) (uint64) + if len("sector_number") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_number\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_number"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_number")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorNumber)); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current_sector") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_sector\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_sector"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current_sector")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous_sector") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_sector\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_sector"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous_sector")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *SectorChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = SectorChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SectorChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.SectorNumber (uint64) (uint64) + case "sector_number": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorNumber = uint64(extra) + + } + // t.Current (typegen.Deferred) (struct) + case "current_sector": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous_sector": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *InfoChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Info (typegen.Deferred) (struct) + if len("info") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"info\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("info")); err != nil { + return err + } + + if err := t.Info.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *InfoChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = InfoChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("InfoChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Info (typegen.Deferred) (struct) + case "info": + + { + + t.Info = new(cbg.Deferred) + + if err := t.Info.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.SectorStatus (cid.Cid) (struct) + if len("sector_status") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_status\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_status"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_status")); err != nil { + return err + } + + if t.SectorStatus == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.SectorStatus); err != nil { + return xerrors.Errorf("failed to write cid field t.SectorStatus: %w", err) + } + } + + // t.Info (cid.Cid) (struct) + if len("info") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"info\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("info")); err != nil { + return err + } + + if t.Info == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + } + + // t.PreCommits (cid.Cid) (struct) + if len("pre_commits") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"pre_commits\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commits"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("pre_commits")); err != nil { + return err + } + + if t.PreCommits == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.PreCommits); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommits: %w", err) + } + } + + // t.Sectors (cid.Cid) (struct) + if len("sectors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sectors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sectors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sectors")); err != nil { + return err + } + + if t.Sectors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Sectors); err != nil { + return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.SectorStatus (cid.Cid) (struct) + case "sector_status": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SectorStatus: %w", err) + } + + t.SectorStatus = &c + } + + } + // t.Info (cid.Cid) (struct) + case "info": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = &c + } + + } + // t.PreCommits (cid.Cid) (struct) + case "pre_commits": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommits: %w", err) + } + + t.PreCommits = &c + } + + } + // t.Sectors (cid.Cid) (struct) + case "sectors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) + } + + t.Sectors = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/minerdiff/v1/infos.go b/pkg/extract/actors/minerdiff/v1/infos.go new file mode 100644 index 000000000..b83364526 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v1/infos.go @@ -0,0 +1,92 @@ +package v1 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*InfoChange)(nil) + +type InfoChange struct { + Info *typegen.Deferred `cborgen:"info"` + Change core.ChangeType `cborgen:"change"` +} + +const KindMinerInfo = "miner_info" + +func (i *InfoChange) Kind() actors.ActorStateKind { + return KindMinerInfo +} + +var _ actors.ActorDiffMethods = (*Info)(nil) + +type Info struct{} + +func (i Info) Type() string { + return KindMinerInfo +} + +func (Info) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerInfo, zap.Inline(act), "duration", time.Since(start)) + }() + return InfoDiff(ctx, api, act) +} + +type DiffInfoAPI interface { + Store() adt.Store + ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) + MinerLoad(store adt.Store, act *types.Actor) (miner.State, error) + Actor(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) +} + +// separate method for testing purposes + +func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.Change) (*InfoChange, error) { + // was removed, no new info + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + + currentMiner, err := api.MinerLoad(api.Store(), act.Current) + if err != nil { + return nil, err + } + infoBytes, err := api.ChainReadObj(ctx, currentMiner.InfoCid()) + if err != nil { + return nil, err + } + // was added, info is new + if act.Type == core.ChangeTypeAdd { + return &InfoChange{ + Info: &typegen.Deferred{Raw: infoBytes}, + Change: core.ChangeTypeAdd, + }, nil + } + + executedMiner, err := api.MinerLoad(api.Store(), act.Executed) + if err != nil { + return nil, err + } + // wasn't modified + if executedMiner.InfoCid().Equals(currentMiner.InfoCid()) { + return nil, nil + } + return &InfoChange{ + Info: &typegen.Deferred{Raw: infoBytes}, + Change: core.ChangeTypeModify, + }, nil +} diff --git a/pkg/extract/actors/minerdiff/v1/load.go b/pkg/extract/actors/minerdiff/v1/load.go new file mode 100644 index 000000000..46b41c7d4 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v1/load.go @@ -0,0 +1,33 @@ +package v1 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/miner" +) + +var MinerStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return miner.Load(store, act) +} + +var MinerPreCommitMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + minerState := m.(miner.State) + perCommitMap, err := minerState.PrecommitsMap() + if err != nil { + return nil, nil, err + } + return perCommitMap, &adt.MapOpts{ + Bitwidth: minerState.PrecommitsMapBitWidth(), + HashFunc: minerState.PrecommitsMapHashFunction(), + }, nil +} + +var MinerSectorArrayLoader = func(m interface{}) (adt.Array, int, error) { + minerState := m.(miner.State) + sectorArray, err := minerState.SectorsArray() + if err != nil { + return nil, -1, err + } + return sectorArray, minerState.SectorsAmtBitwidth(), nil +} diff --git a/pkg/extract/actors/minerdiff/v1/precommits.go b/pkg/extract/actors/minerdiff/v1/precommits.go new file mode 100644 index 000000000..694bcffb2 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v1/precommits.go @@ -0,0 +1,85 @@ +package v1 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*PreCommitChangeList)(nil) + +var _ abi.Keyer = (*PreCommitChange)(nil) + +type PreCommitChange struct { + SectorNumber []byte `cborgen:"sector_number"` + Current *typegen.Deferred `cborgen:"current_pre_commit"` + Previous *typegen.Deferred `cborgen:"previous_pre_commit"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *PreCommitChange) Key() string { + return core.StringKey(t.SectorNumber).Key() +} + +type PreCommitChangeList []*PreCommitChange + +const KindMinerPreCommit = "miner_precommit" + +func (p PreCommitChangeList) Kind() actors.ActorStateKind { + return KindMinerPreCommit +} + +func (p PreCommitChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type PreCommit struct{} + +func (c PreCommit) Type() string { + return KindMinerPreCommit +} + +func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerPreCommit, zap.Inline(act), "duration", time.Since(start)) + }() + return PreCommitDiff(ctx, api, act) +} + +func PreCommitDiff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, MinerStateLoader, MinerPreCommitMapLoader) + if err != nil { + return nil, err + } + out := make(PreCommitChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &PreCommitChange{ + SectorNumber: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/minerdiff/v1/sector_status.go b/pkg/extract/actors/minerdiff/v1/sector_status.go new file mode 100644 index 000000000..49ee1b81e --- /dev/null +++ b/pkg/extract/actors/minerdiff/v1/sector_status.go @@ -0,0 +1,194 @@ +package v1 + +import ( + "context" + "fmt" + "time" + + "github.com/filecoin-project/go-bitfield" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*SectorStatusChange)(nil) + +type SectorStatusChange struct { + // Removed sectors this epoch + Removed bitfield.BitField `cborgen:"removed"` + // Recovering sectors this epoch + Recovering bitfield.BitField `cborgen:"recovering"` + // Faulted sectors this epoch + Faulted bitfield.BitField `cborgen:"faulted"` + // Recovered sectors this epoch + Recovered bitfield.BitField `cborgen:"recovered"` +} + +const KindMinerSectorStatus = "miner_sector_status" + +func (s SectorStatusChange) Kind() actors.ActorStateKind { + return KindMinerSectorStatus +} + +type SectorStatus struct{} + +func (s SectorStatus) Type() string { + return KindMinerSectorStatus +} + +func (SectorStatus) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerSectorStatus, zap.Inline(act), "duration", time.Since(start)) + }() + // miner must have current and executed state for its sectors to have status. + if act.Type == core.ChangeTypeAdd || act.Type == core.ChangeTypeRemove { + return nil, nil + } + child, err := api.MinerLoad(api.Store(), act.Current) + if err != nil { + return nil, err + } + parent, err := api.MinerLoad(api.Store(), act.Executed) + if err != nil { + return nil, err + } + return DiffMinerSectorStates(ctx, child, parent) +} + +// DiffMinerSectorStates loads the SectorStates for the current and parent miner states in parallel from `extState`. +// Then compares current and parent SectorStates to produce a SectorStateEvents structure containing all sectors that are +// removed, recovering, faulted, and recovered for the state transition from parent miner state to current miner state. +func DiffMinerSectorStates(ctx context.Context, child, parent miner.State) (*SectorStatusChange, error) { + var ( + previous, current *SectorStates + err error + ) + + // load previous and current miner sector states in parallel + grp, grpCtx := errgroup.WithContext(ctx) + grp.Go(func() error { + previous, err = LoadSectorState(grpCtx, parent) + if err != nil { + return fmt.Errorf("loading previous sector states %w", err) + } + return nil + }) + grp.Go(func() error { + current, err = LoadSectorState(grpCtx, child) + if err != nil { + return fmt.Errorf("loading current sector states %w", err) + } + return nil + }) + // if either load operation fails abort + if err := grp.Wait(); err != nil { + return nil, err + } + + // previous live sector minus current live sectors are sectors removed this epoch. + removed, err := bitfield.SubtractBitField(previous.Live, current.Live) + if err != nil { + return nil, fmt.Errorf("comparing previous live sectors to current live sectors %w", err) + } + + // current recovering sectors minus previous recovering sectors are sectors recovering this epoch. + recovering, err := bitfield.SubtractBitField(current.Recovering, previous.Recovering) + if err != nil { + return nil, fmt.Errorf("comparing current recovering sectors to previous recovering sectors %w", err) + } + + // current faulty sectors minus previous faulty sectors are sectors faulted this epoch. + faulted, err := bitfield.SubtractBitField(current.Faulty, previous.Faulty) + if err != nil { + return nil, fmt.Errorf("comparing current faulty sectors to previous faulty sectors %w", err) + } + + // previous faulty sectors matching (intersect) active sectors are sectors recovered this epoch. + recovered, err := bitfield.IntersectBitField(previous.Faulty, current.Active) + if err != nil { + return nil, fmt.Errorf("comparing previous faulty sectors to current active sectors %w", err) + } + + return &SectorStatusChange{ + Removed: removed, + Recovering: recovering, + Faulted: faulted, + Recovered: recovered, + }, nil + +} + +// SectorStates contains a set of bitfields for active, live, fault, and recovering sectors. +type SectorStates struct { + // Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power. + Active bitfield.BitField + // Live sectors are those that are not terminated (but may be faulty). + Live bitfield.BitField + // Faulty contains a subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt). + Faulty bitfield.BitField + // Recovering contains a subset of faulty sectors expected to recover on next PoSt. + Recovering bitfield.BitField +} + +// LoadSectorState loads all sectors from a miners partitions and returns a SectorStates structure containing individual +// bitfields for all active, live, faulty and recovering sector. +func LoadSectorState(ctx context.Context, state miner.State) (*SectorStates, error) { + var activeSectors []bitfield.BitField + var liveSectors []bitfield.BitField + var faultySectors []bitfield.BitField + var recoveringSectors []bitfield.BitField + + // iterate the sector states + if err := state.ForEachDeadline(func(_ uint64, dl miner.Deadline) error { + return dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + active, err := part.ActiveSectors() + if err != nil { + return err + } + activeSectors = append(activeSectors, active) + + live, err := part.LiveSectors() + if err != nil { + return err + } + liveSectors = append(liveSectors, live) + + faulty, err := part.FaultySectors() + if err != nil { + return err + } + faultySectors = append(faultySectors, faulty) + + recovering, err := part.RecoveringSectors() + if err != nil { + return err + } + recoveringSectors = append(recoveringSectors, recovering) + + return nil + }) + }); err != nil { + return nil, err + } + var err error + sectorStates := &SectorStates{} + if sectorStates.Active, err = bitfield.MultiMerge(activeSectors...); err != nil { + return nil, err + } + if sectorStates.Live, err = bitfield.MultiMerge(liveSectors...); err != nil { + return nil, err + } + if sectorStates.Faulty, err = bitfield.MultiMerge(faultySectors...); err != nil { + return nil, err + } + if sectorStates.Recovering, err = bitfield.MultiMerge(recoveringSectors...); err != nil { + return nil, err + } + + return sectorStates, nil +} diff --git a/pkg/extract/actors/minerdiff/v1/sectors.go b/pkg/extract/actors/minerdiff/v1/sectors.go new file mode 100644 index 000000000..1a755e6ec --- /dev/null +++ b/pkg/extract/actors/minerdiff/v1/sectors.go @@ -0,0 +1,78 @@ +package v1 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*SectorChangeList)(nil) + +type SectorChange struct { + SectorNumber uint64 `cborgen:"sector_number"` + Current *typegen.Deferred `cborgen:"current_sector"` + Previous *typegen.Deferred `cborgen:"previous_sector"` + Change core.ChangeType `cborgen:"change"` +} + +type SectorChangeList []*SectorChange + +func (s SectorChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range s { + if err := node.Put(abi.UIntKey(l.SectorNumber), l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +const KindMinerSector = "miner_sector" + +func (s SectorChangeList) Kind() actors.ActorStateKind { + return KindMinerSector +} + +type Sectors struct{} + +func (s Sectors) Type() string { + return KindMinerSector +} + +func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerSector, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffSectors(ctx, api, act) +} + +func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + arrayChange, err := generic.DiffActorArray(ctx, api, act, MinerStateLoader, MinerSectorArrayLoader) + if err != nil { + return nil, err + } + out := make(SectorChangeList, len(arrayChange)) + for i, change := range arrayChange { + out[i] = &SectorChange{ + SectorNumber: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/minerdiff/v1/state.go b/pkg/extract/actors/minerdiff/v1/state.go new file mode 100644 index 000000000..ff69bbfb8 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v1/state.go @@ -0,0 +1,166 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +var log = logging.Logger("extract/actors/miner") + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case *InfoChange: + stateDiff.InfoChange = v + case SectorChangeList: + stateDiff.SectorChanges = v + case PreCommitChangeList: + stateDiff.PreCommitChanges = v + case *SectorStatusChange: + stateDiff.SectorStatusChanges = v + default: + return nil, fmt.Errorf("unknown state change %T", v) + } + } + + return stateDiff, nil +} + +type StateDiffResult struct { + InfoChange *InfoChange + PreCommitChanges PreCommitChangeList + SectorChanges SectorChangeList + SectorStatusChanges *SectorStatusChange +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) { + out := &StateChange{} + + if sectors := sd.SectorChanges; sectors != nil { + root, err := sectors.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Sectors = &root + } + + if precommits := sd.PreCommitChanges; precommits != nil { + root, err := precommits.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.PreCommits = &root + } + + if info := sd.InfoChange; info != nil { + c, err := s.Put(ctx, info) + if err != nil { + return nil, err + } + out.Info = &c + } + + if sectorstatus := sd.SectorStatusChanges; sectorstatus != nil { + c, err := s.Put(ctx, sectorstatus) + if err != nil { + return nil, err + } + out.SectorStatus = &c + } + return out, nil +} + +type StateChange struct { + // SectorStatus is the sectors whose status changed for this miner or empty. + SectorStatus *cid.Cid `cborgen:"sector_status"` + // Info is the cid of the miner change info that changed for this miner or empty. + Info *cid.Cid `cborgen:"info"` + // PreCommits is an HAMT of the pre commits that changed for this miner. + PreCommits *cid.Cid `cborgen:"pre_commits"` + // Sectors is an HAMT of the sectors that changed for this miner. + Sectors *cid.Cid `cborgen:"sectors"` +} + +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + InfoChange: nil, + SectorStatusChanges: nil, + SectorChanges: SectorChangeList{}, + PreCommitChanges: PreCommitChangeList{}, + } + + // + // SectorChangeList + if sc.Sectors != nil { + sectorMap, err := adt.AsMap(s, *sc.Sectors, 5) + if err != nil { + return nil, err + } + + sectors := SectorChangeList{} + sectorChange := new(SectorChange) + if err := sectorMap.ForEach(sectorChange, func(sectorNumber string) error { + val := new(SectorChange) + *val = *sectorChange + sectors = append(sectors, val) + return nil + }); err != nil { + return nil, err + } + out.SectorChanges = sectors + } + + // + // PrecommitChangeList + + if sc.PreCommits != nil { + precommitMap, err := adt.AsMap(s, *sc.PreCommits, 5) + if err != nil { + return nil, err + } + + precommits := PreCommitChangeList{} + precommitChange := new(PreCommitChange) + if err := precommitMap.ForEach(precommitChange, func(sectorNumber string) error { + val := new(PreCommitChange) + *val = *precommitChange + precommits = append(precommits, val) + return nil + }); err != nil { + return nil, err + } + out.PreCommitChanges = precommits + } + + // + // Info + if sc.Info != nil { + info := new(InfoChange) + if err := s.Get(ctx, *sc.Info, info); err != nil { + return nil, err + } + out.InfoChange = info + } + + // + // SectorStatus + + if sc.SectorStatus != nil { + status := new(SectorStatusChange) + if err := s.Get(ctx, *sc.SectorStatus, status); err != nil { + return nil, err + } + out.SectorStatusChanges = status + } + + return out, nil +} diff --git a/pkg/extract/actors/minerdiff/version.go b/pkg/extract/actors/minerdiff/version.go new file mode 100644 index 000000000..4bbcac9cc --- /dev/null +++ b/pkg/extract/actors/minerdiff/version.go @@ -0,0 +1,24 @@ +package minerdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + switch av { + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return []actors.ActorDiffMethods{ + v1.Info{}, + v1.PreCommit{}, + v1.SectorStatus{}, + v1.Sectors{}, + }, v1.ActorStateChangeHandler, nil + } + return nil, nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/powerdiff/v1/cbor_gen.go b/pkg/extract/actors/powerdiff/v1/cbor_gen.go new file mode 100644 index 000000000..c2e3a33ef --- /dev/null +++ b/pkg/extract/actors/powerdiff/v1/cbor_gen.go @@ -0,0 +1,319 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v1 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Claims (cid.Cid) (struct) + if len("claims") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claims\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claims"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claims")); err != nil { + return err + } + + if t.Claims == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Claims); err != nil { + return xerrors.Errorf("failed to write cid field t.Claims: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Claims (cid.Cid) (struct) + case "claims": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Claims: %w", err) + } + + t.Claims = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Miner ([]uint8) (slice) + if len("miner") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"miner\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miner"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("miner")); err != nil { + return err + } + + if len(t.Miner) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Miner was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Miner))); err != nil { + return err + } + + if _, err := cw.Write(t.Miner[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ClaimsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ClaimsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Miner ([]uint8) (slice) + case "miner": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Miner: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Miner = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Miner[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/powerdiff/v1/claims.go b/pkg/extract/actors/powerdiff/v1/claims.go new file mode 100644 index 000000000..124eaca89 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v1/claims.go @@ -0,0 +1,78 @@ +package v1 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/power") + +type ClaimsChange struct { + Miner []byte `cborgen:"miner"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +type ClaimsChangeList []*ClaimsChange + +const KindPowerClaims = "power_claims" + +func (c ClaimsChangeList) Kind() actors.ActorStateKind { + return KindPowerClaims +} + +func (p ClaimsChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(core.StringKey(l.Miner), l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type Claims struct{} + +func (c Claims) Type() string { + return KindPowerClaims +} + +func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindPowerClaims, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffClaims(ctx, api, act) +} + +func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, PowerStateLoader, PowerClaimsMapLoader) + if err != nil { + return nil, err + } + out := make(ClaimsChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &ClaimsChange{ + Miner: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/powerdiff/v1/load.go b/pkg/extract/actors/powerdiff/v1/load.go new file mode 100644 index 000000000..f49b979ef --- /dev/null +++ b/pkg/extract/actors/powerdiff/v1/load.go @@ -0,0 +1,24 @@ +package v1 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/power" +) + +var PowerStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return power.Load(store, act) +} + +var PowerClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + powerState := m.(power.State) + claimsMap, err := powerState.ClaimsMap() + if err != nil { + return nil, nil, err + } + return claimsMap, &adt.MapOpts{ + Bitwidth: powerState.ClaimsMapBitWidth(), + HashFunc: powerState.ClaimsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/powerdiff/v1/state.go b/pkg/extract/actors/powerdiff/v1/state.go new file mode 100644 index 000000000..90a71a887 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v1/state.go @@ -0,0 +1,70 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case ClaimsChangeList: + stateDiff.ClaimsChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) + } + } + return stateDiff, nil +} + +type StateDiffResult struct { + ClaimsChanges ClaimsChangeList +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { + out := &StateChange{} + if claims := sd.ClaimsChanges; claims != nil { + root, err := claims.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Claims = &root + } + return out, nil +} + +type StateChange struct { + Claims *cid.Cid `cborgen:"claims"` +} + +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ClaimsChanges: ClaimsChangeList{}} + + if sc.Claims != nil { + claimMap, err := adt.AsMap(s, *sc.Claims, 5) + if err != nil { + return nil, err + } + + claims := ClaimsChangeList{} + claimChange := new(ClaimsChange) + if err := claimMap.ForEach(claimChange, func(key string) error { + val := new(ClaimsChange) + *val = *claimChange + claims = append(claims, val) + return nil + }); err != nil { + return nil, err + } + out.ClaimsChanges = claims + } + return out, nil +} diff --git a/pkg/extract/actors/powerdiff/version.go b/pkg/extract/actors/powerdiff/version.go new file mode 100644 index 000000000..31e8d53fb --- /dev/null +++ b/pkg/extract/actors/powerdiff/version.go @@ -0,0 +1,19 @@ +package powerdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + switch av { + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return []actors.ActorDiffMethods{v1.Claims{}}, v1.ActorStateChangeHandler, nil + } + return nil, nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/rawdiff/actor.go b/pkg/extract/actors/rawdiff/actor.go new file mode 100644 index 000000000..55bd19847 --- /dev/null +++ b/pkg/extract/actors/rawdiff/actor.go @@ -0,0 +1,87 @@ +package rawdiff + +import ( + "context" + "fmt" + "time" + + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/actor") + +type ActorChange struct { + Actor *types.Actor `cborgen:"actor"` + Current []byte `cborgen:"current_state"` + Previous []byte `cborgen:"previous_state"` + Change core.ChangeType `cborgen:"change"` +} + +const KindActorChange = "actor_change" + +func (a *ActorChange) Kind() actors.ActorStateKind { + return KindActorChange +} + +type Actor struct{} + +func (Actor) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindActorChange, zap.Inline(act), "duration", time.Since(start)) + }() + return ActorDiff(ctx, api, act) +} + +func (Actor) Type() string { + return KindActorChange +} + +func ActorDiff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + switch act.Type { + case core.ChangeTypeAdd: + currentState, err := api.ChainReadObj(ctx, act.Current.Head) + if err != nil { + return nil, err + } + return &ActorChange{ + Actor: act.Current, + Current: currentState, + Previous: nil, + Change: act.Type, + }, nil + case core.ChangeTypeRemove: + executedState, err := api.ChainReadObj(ctx, act.Executed.Head) + if err != nil { + return nil, err + } + return &ActorChange{ + Actor: act.Executed, + Current: nil, + Previous: executedState, + Change: act.Type, + }, nil + case core.ChangeTypeModify: + currentState, err := api.ChainReadObj(ctx, act.Current.Head) + if err != nil { + return nil, err + } + executedState, err := api.ChainReadObj(ctx, act.Executed.Head) + if err != nil { + return nil, err + } + return &ActorChange{ + Actor: act.Current, + Current: currentState, + Previous: executedState, + Change: act.Type, + }, nil + } + return nil, fmt.Errorf("unknown actor change type %s", act.Type) +} diff --git a/pkg/extract/actors/rawdiff/cbor_gen.go b/pkg/extract/actors/rawdiff/cbor_gen.go new file mode 100644 index 000000000..cc097b1bd --- /dev/null +++ b/pkg/extract/actors/rawdiff/cbor_gen.go @@ -0,0 +1,332 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package rawdiff + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + types "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *ActorChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Actor (types.Actor) (struct) + if len("actor") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"actor\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actor"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("actor")); err != nil { + return err + } + + if err := t.Actor.MarshalCBOR(cw); err != nil { + return err + } + + // t.Current ([]uint8) (slice) + if len("current_state") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_state\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_state"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current_state")); err != nil { + return err + } + + if len(t.Current) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Current was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Current))); err != nil { + return err + } + + if _, err := cw.Write(t.Current[:]); err != nil { + return err + } + + // t.Previous ([]uint8) (slice) + if len("previous_state") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_state\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_state"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous_state")); err != nil { + return err + } + + if len(t.Previous) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Previous was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Previous))); err != nil { + return err + } + + if _, err := cw.Write(t.Previous[:]); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ActorChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Actor (types.Actor) (struct) + case "actor": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Actor = new(types.Actor) + if err := t.Actor.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Actor pointer: %w", err) + } + } + + } + // t.Current ([]uint8) (slice) + case "current_state": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Current: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Current = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Current[:]); err != nil { + return err + } + // t.Previous ([]uint8) (slice) + case "previous_state": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Previous: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Previous = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Previous[:]); err != nil { + return err + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.ActorState (cid.Cid) (struct) + if len("actors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"actors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("actors")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.ActorState); err != nil { + return xerrors.Errorf("failed to write cid field t.ActorState: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.ActorState (cid.Cid) (struct) + case "actors": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ActorState: %w", err) + } + + t.ActorState = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/rawdiff/state.go b/pkg/extract/actors/rawdiff/state.go new file mode 100644 index 000000000..7d5f0998e --- /dev/null +++ b/pkg/extract/actors/rawdiff/state.go @@ -0,0 +1,37 @@ +package rawdiff + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, change := range changes { + switch v := change.(type) { + case *ActorChange: + stateDiff.ActorStateChanges = v + default: + return nil, fmt.Errorf("unhandled change type: %T", v) + } + } + return stateDiff, nil +} + +type StateDiffResult struct { + ActorStateChanges *ActorChange +} + +func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { + return sdr.ActorStateChanges, nil +} + +type StateChange struct { + ActorState cid.Cid `cborgen:"actors"` +} diff --git a/pkg/extract/actors/rawdiff/version.go b/pkg/extract/actors/rawdiff/version.go new file mode 100644 index 000000000..831371e95 --- /dev/null +++ b/pkg/extract/actors/rawdiff/version.go @@ -0,0 +1,11 @@ +package rawdiff + +import ( + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + return []actors.ActorDiffMethods{Actor{}}, ActorStateChangeHandler, nil +} diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go new file mode 100644 index 000000000..441547669 --- /dev/null +++ b/pkg/extract/actors/types.go @@ -0,0 +1,77 @@ +package actors + +import ( + "context" + "time" + + "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + typegen "github.com/whyrusleeping/cbor-gen" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" + + "github.com/filecoin-project/lily/chain/actors/builtin" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/tasks" +) + +type ExtractionReport struct { + Current *types.TipSet + Executed *types.TipSet + NetworkVersion int // version of the network + ActorVersion int // version of the actor being extracted + ExtractorVersion int // version of the extraction logic + ExtractorKind string // miner/market/init/power/etc. + DifferKind string // miner_info/miner_sectors/market_claims/power_claims/etc. + ChangeType core.ChangeType // Added, Modified, Removed + StartTime time.Time // when the differ started execution + Duration time.Duration // how long the differ took to run +} + +type DiffResult interface { + MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) +} + +type ActorDiff interface { + State(ctx context.Context, api tasks.DataSource, act *Change) (DiffResult, error) +} + +type ActorDiffMethods interface { + Diff(ctx context.Context, api tasks.DataSource, act *Change) (ActorStateChange, error) + Type() string +} + +type ActorStateKind string + +type ActorStateChange interface { + Kind() ActorStateKind +} + +type Change struct { + Address address.Address + + Executed *types.Actor + ExeVersion actortypes.Version + + Current *types.Actor + CurVersion actortypes.Version + + Type core.ChangeType +} + +func (a Change) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("address", a.Address.String()), + attribute.String("type", builtin.ActorNameByCode(a.Current.Code)), + attribute.String("change", a.Type.String()), + } +} + +func (a Change) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range a.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} diff --git a/pkg/extract/actors/verifregdiff/v1/cbor_gen.go b/pkg/extract/actors/verifregdiff/v1/cbor_gen.go new file mode 100644 index 000000000..adfef2e46 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v1/cbor_gen.go @@ -0,0 +1,555 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v1 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Verifiers (cid.Cid) (struct) + if len("verifiers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifiers\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifiers"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifiers")); err != nil { + return err + } + + if t.Verifiers == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Verifiers); err != nil { + return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) + } + } + + // t.Clients (cid.Cid) (struct) + if len("clients") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"clients\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("clients"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("clients")); err != nil { + return err + } + + if t.Clients == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Clients); err != nil { + return xerrors.Errorf("failed to write cid field t.Clients: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Verifiers (cid.Cid) (struct) + case "verifiers": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) + } + + t.Verifiers = &c + } + + } + // t.Clients (cid.Cid) (struct) + case "clients": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Clients: %w", err) + } + + t.Clients = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ClientsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Client ([]uint8) (slice) + if len("client") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"client\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("client"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("client")); err != nil { + return err + } + + if len(t.Client) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Client was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Client))); err != nil { + return err + } + + if _, err := cw.Write(t.Client[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ClientsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ClientsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ClientsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Client ([]uint8) (slice) + case "client": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Client: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Client = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Client[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *VerifiersChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Verifier ([]uint8) (slice) + if len("verifier") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifier\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifier"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifier")); err != nil { + return err + } + + if len(t.Verifier) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Verifier was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Verifier))); err != nil { + return err + } + + if _, err := cw.Write(t.Verifier[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *VerifiersChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = VerifiersChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("VerifiersChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Verifier ([]uint8) (slice) + case "verifier": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Verifier: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Verifier = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Verifier[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/verifregdiff/v1/clients.go b/pkg/extract/actors/verifregdiff/v1/clients.go new file mode 100644 index 000000000..50cf1bc9e --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v1/clients.go @@ -0,0 +1,87 @@ +package v1 + +import ( + "context" + "time" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +// TODO this log name could be confusing since its reused by subsequent versions +var log = logging.Logger("lily/extract/actors/verifreg/v0") + +// TODO add cbor gen tags +type ClientsChange struct { + Client []byte `cborgen:"client"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *ClientsChange) Key() string { + return core.StringKey(t.Client).Key() +} + +type ClientsChangeList []*ClientsChange + +const KindVerifregClients = "verifreg_clients" + +func (v ClientsChangeList) Kind() actors.ActorStateKind { + return KindVerifregClients +} + +func (v ClientsChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range v { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type Clients struct{} + +func (c Clients) Type() string { + return KindVerifregClients +} + +func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregClients, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffClients(ctx, api, act) +} + +func DiffClients(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiregClientsMapLoader) + if err != nil { + return nil, err + } + + out := make(ClientsChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &ClientsChange{ + Client: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil + +} diff --git a/pkg/extract/actors/verifregdiff/v1/load.go b/pkg/extract/actors/verifregdiff/v1/load.go new file mode 100644 index 000000000..3e88d9e8e --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v1/load.go @@ -0,0 +1,37 @@ +package v1 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" +) + +// VerifregStateLoader returns the verifiergistry actor state for `act`. +var VerifregStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return verifreg.Load(store, act) +} + +var VerifiregVerifiersMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + verifierMap, err := verifregState.VerifiersMap() + if err != nil { + return nil, nil, err + } + return verifierMap, &adt.MapOpts{ + Bitwidth: verifregState.VerifiersMapBitWidth(), + HashFunc: verifregState.VerifiersMapHashFunction(), + }, nil +} + +var VerifiregClientsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + clientsMap, err := verifregState.VerifiedClientsMap() + if err != nil { + return nil, nil, err + } + return clientsMap, &adt.MapOpts{ + Bitwidth: verifregState.VerifiedClientsMapBitWidth(), + HashFunc: verifregState.VerifiedClientsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/verifregdiff/v1/state.go b/pkg/extract/actors/verifregdiff/v1/state.go new file mode 100644 index 000000000..6712f2c43 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v1/state.go @@ -0,0 +1,101 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case ClientsChangeList: + stateDiff.ClientChanges = v + case VerifiersChangeList: + stateDiff.VerifierChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) + } + } + return stateDiff, nil +} + +type StateDiffResult struct { + VerifierChanges VerifiersChangeList + ClientChanges ClientsChangeList +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) { + out := &StateChange{} + + if clients := sd.ClientChanges; clients != nil { + root, err := clients.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Clients = &root + } + + if verifiers := sd.VerifierChanges; verifiers != nil { + root, err := verifiers.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Verifiers = &root + } + return out, nil +} + +type StateChange struct { + Verifiers *cid.Cid `cborgen:"verifiers"` + Clients *cid.Cid `cborgen:"clients"` +} + +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + VerifierChanges: VerifiersChangeList{}, + ClientChanges: ClientsChangeList{}, + } + + if sc.Verifiers != nil { + verifierMap, err := adt.AsMap(s, *sc.Verifiers, 5) + if err != nil { + return nil, err + } + + verifierChange := new(VerifiersChange) + if err := verifierMap.ForEach(verifierChange, func(key string) error { + val := new(VerifiersChange) + *val = *verifierChange + out.VerifierChanges = append(out.VerifierChanges, val) + return nil + }); err != nil { + return nil, err + } + } + + if sc.Clients != nil { + clientsMap, err := adt.AsMap(s, *sc.Clients, 5) + if err != nil { + return nil, err + } + + clientChange := new(ClientsChange) + if err := clientsMap.ForEach(clientChange, func(key string) error { + val := new(ClientsChange) + *val = *clientChange + out.ClientChanges = append(out.ClientChanges, val) + return nil + }); err != nil { + return nil, err + } + } + return out, nil +} diff --git a/pkg/extract/actors/verifregdiff/v1/verifiers.go b/pkg/extract/actors/verifregdiff/v1/verifiers.go new file mode 100644 index 000000000..a930cb985 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v1/verifiers.go @@ -0,0 +1,81 @@ +package v1 + +import ( + "context" + "time" + + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +// TODO add cbor gen tags +type VerifiersChange struct { + Verifier []byte `cborgen:"verifier"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *VerifiersChange) Key() string { + return core.StringKey(t.Verifier).Key() +} + +type VerifiersChangeList []*VerifiersChange + +const KindVerifregVerifiers = "verifreg_verifiers" + +func (v VerifiersChangeList) Kind() actors.ActorStateKind { + return KindVerifregVerifiers +} + +func (v VerifiersChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range v { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +type Verifiers struct{} + +func (v Verifiers) Type() string { + return KindVerifregVerifiers +} + +func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregVerifiers, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffVerifiers(ctx, api, act) +} + +func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiregVerifiersMapLoader) + if err != nil { + return nil, err + } + out := make(VerifiersChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &VerifiersChange{ + Verifier: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/verifregdiff/v2/allocations.go b/pkg/extract/actors/verifregdiff/v2/allocations.go new file mode 100644 index 000000000..7760c8077 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/allocations.go @@ -0,0 +1,131 @@ +package v2 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + "github.com/filecoin-project/lily/tasks" +) + +// TODO add cbor gen tags +type AllocationsChange struct { + Client []byte `cborgen:"provider"` + ClaimID []byte `cborgen:"claimID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +type AllocationsChangeMap map[address.Address][]*AllocationsChange + +const KindVerifregAllocations = "verifreg_allocations" + +func (c AllocationsChangeMap) Kind() actors.ActorStateKind { + return KindVerifregAllocations +} + +func (c AllocationsChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + topNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for client, changes := range c { + innerNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, change := range changes { + if err := innerNode.Put(core.StringKey(change.ClaimID), change); err != nil { + return cid.Undef, err + } + } + innerRoot, err := innerNode.Root() + if err != nil { + return cid.Undef, err + } + if err := topNode.Put(abi.IdAddrKey(client), typegen.CborCid(innerRoot)); err != nil { + return cid.Undef, err + } + } + return topNode.Root() +} + +type Allocations struct{} + +func (a Allocations) Type() string { + return KindVerifregAllocations +} + +func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregAllocations, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffAllocations(ctx, api, act) +} + +func DiffAllocations(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifregAllocationMapLoader) + if err != nil { + return nil, err + } + out := make(AllocationsChangeMap) + for _, change := range mapChange { + clientId, err := abi.ParseUIntKey(string(change.Key)) + if err != nil { + return nil, err + } + clientAddress, err := address.NewIDAddress(clientId) + if err != nil { + return nil, err + } + clientMapChanges, err := diffClientMap(ctx, api, act, clientAddress, change.Key) + if err != nil { + return nil, err + } + out[clientAddress] = clientMapChanges + } + return out, nil +} + +func diffClientMap(ctx context.Context, api tasks.DataSource, act *actors.Change, clientAddress address.Address, clientKey []byte) ([]*AllocationsChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := i.(verifreg.State) + clientAllocationMap, err := verifregState.AllocationMapForClient(clientAddress) + if err != nil { + return nil, nil, err + } + return clientAllocationMap, &adt.MapOpts{ + Bitwidth: verifregState.AllocationsMapBitWidth(), + HashFunc: verifregState.AllocationsMapHashFunction(), + }, nil + }) + if err != nil { + return nil, err + } + out := make([]*AllocationsChange, 0, len(mapChange)) + for _, change := range mapChange { + out = append(out, &AllocationsChange{ + Client: clientKey, + ClaimID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + }) + } + return out, nil +} diff --git a/pkg/extract/actors/verifregdiff/v2/cbor_gen.go b/pkg/extract/actors/verifregdiff/v2/cbor_gen.go new file mode 100644 index 000000000..d836c2eb6 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/cbor_gen.go @@ -0,0 +1,692 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v2 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Verifiers (cid.Cid) (struct) + if len("verifiers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifiers\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifiers"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifiers")); err != nil { + return err + } + + if t.Verifiers == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Verifiers); err != nil { + return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) + } + } + + // t.Claims (cid.Cid) (struct) + if len("claims") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claims\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claims"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claims")); err != nil { + return err + } + + if t.Claims == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Claims); err != nil { + return xerrors.Errorf("failed to write cid field t.Claims: %w", err) + } + } + + // t.Allocations (cid.Cid) (struct) + if len("allocations") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"allocations\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allocations"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("allocations")); err != nil { + return err + } + + if t.Allocations == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Allocations); err != nil { + return xerrors.Errorf("failed to write cid field t.Allocations: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Verifiers (cid.Cid) (struct) + case "verifiers": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) + } + + t.Verifiers = &c + } + + } + // t.Claims (cid.Cid) (struct) + case "claims": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Claims: %w", err) + } + + t.Claims = &c + } + + } + // t.Allocations (cid.Cid) (struct) + case "allocations": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Allocations: %w", err) + } + + t.Allocations = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Provider ([]uint8) (slice) + if len("provider") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"provider\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("provider"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("provider")); err != nil { + return err + } + + if len(t.Provider) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Provider was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Provider))); err != nil { + return err + } + + if _, err := cw.Write(t.Provider[:]); err != nil { + return err + } + + // t.ClaimID ([]uint8) (slice) + if len("claimID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claimID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claimID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claimID")); err != nil { + return err + } + + if len(t.ClaimID) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.ClaimID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.ClaimID))); err != nil { + return err + } + + if _, err := cw.Write(t.ClaimID[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ClaimsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ClaimsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Provider ([]uint8) (slice) + case "provider": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Provider: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Provider = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Provider[:]); err != nil { + return err + } + // t.ClaimID ([]uint8) (slice) + case "claimID": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.ClaimID: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.ClaimID = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.ClaimID[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *AllocationsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Client ([]uint8) (slice) + if len("provider") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"provider\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("provider"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("provider")); err != nil { + return err + } + + if len(t.Client) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Client was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Client))); err != nil { + return err + } + + if _, err := cw.Write(t.Client[:]); err != nil { + return err + } + + // t.ClaimID ([]uint8) (slice) + if len("claimID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claimID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claimID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claimID")); err != nil { + return err + } + + if len(t.ClaimID) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.ClaimID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.ClaimID))); err != nil { + return err + } + + if _, err := cw.Write(t.ClaimID[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *AllocationsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = AllocationsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("AllocationsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Client ([]uint8) (slice) + case "provider": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Client: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Client = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Client[:]); err != nil { + return err + } + // t.ClaimID ([]uint8) (slice) + case "claimID": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.ClaimID: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.ClaimID = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.ClaimID[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/verifregdiff/v2/claims.go b/pkg/extract/actors/verifregdiff/v2/claims.go new file mode 100644 index 000000000..7c06ef5eb --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/claims.go @@ -0,0 +1,134 @@ +package v2 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + + adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +// TODO add cbor gen tags +type ClaimsChange struct { + Provider []byte `cborgen:"provider"` + ClaimID []byte `cborgen:"claimID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +type ClaimsChangeMap map[address.Address][]*ClaimsChange + +const KindVerifregClaims = "verifreg_claims" + +func (c ClaimsChangeMap) Kind() actors.ActorStateKind { + return KindVerifregClaims +} + +// returns a HAMT[provider]HAMT[claimID]ClaimsChange +func (c ClaimsChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + topNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for provider, changes := range c { + innerNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, change := range changes { + if err := innerNode.Put(core.StringKey(change.ClaimID), change); err != nil { + return cid.Undef, err + } + } + innerRoot, err := innerNode.Root() + if err != nil { + return cid.Undef, err + } + if err := topNode.Put(abi.IdAddrKey(provider), typegen.CborCid(innerRoot)); err != nil { + return cid.Undef, err + } + } + return topNode.Root() +} + +type Claims struct{} + +func (c Claims) Type() string { + return KindVerifregClaims +} + +func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregClaims, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffClaims(ctx, api, act) +} + +func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifregClaimsMapLoader) + if err != nil { + return nil, err + } + // map change is keyed on provider address with value adt.Map + out := make(ClaimsChangeMap) + for _, change := range mapChange { + providerID, err := abi.ParseUIntKey(string(change.Key)) + if err != nil { + return nil, err + } + providerAddress, err := address.NewIDAddress(providerID) + if err != nil { + return nil, err + } + subMapChanges, err := diffProviderMap(ctx, api, act, providerAddress, change.Key) + if err != nil { + return nil, err + } + out[providerAddress] = subMapChanges + } + return out, nil +} + +func diffProviderMap(ctx context.Context, api tasks.DataSource, act *actors.Change, providerAddress address.Address, providerKey []byte) ([]*ClaimsChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := i.(verifreg.State) + providerClaimMap, err := verifregState.ClaimMapForProvider(providerAddress) + if err != nil { + return nil, nil, err + } + return providerClaimMap, &adt.MapOpts{ + Bitwidth: verifregState.ClaimsMapBitWidth(), + HashFunc: verifregState.ClaimsMapHashFunction(), + }, nil + }) + if err != nil { + return nil, err + } + out := make([]*ClaimsChange, 0, len(mapChange)) + for _, change := range mapChange { + out = append(out, &ClaimsChange{ + Provider: providerKey, + ClaimID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + }) + } + return out, nil +} diff --git a/pkg/extract/actors/verifregdiff/v2/load.go b/pkg/extract/actors/verifregdiff/v2/load.go new file mode 100644 index 000000000..40c397138 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/load.go @@ -0,0 +1,30 @@ +package v2 + +import ( + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" +) + +var VerifregClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + claimsMap, err := verifregState.ClaimsMap() + if err != nil { + return nil, nil, err + } + return claimsMap, &adt.MapOpts{ + Bitwidth: verifregState.ClaimsMapBitWidth(), + HashFunc: verifregState.ClaimsMapHashFunction(), + }, nil +} + +var VerifregAllocationMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + allocationMap, err := verifregState.AllocationsMap() + if err != nil { + return nil, nil, err + } + return allocationMap, &adt.MapOpts{ + Bitwidth: verifregState.AllocationsMapBitWidth(), + HashFunc: verifregState.AllocationsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/verifregdiff/v2/state.go b/pkg/extract/actors/verifregdiff/v2/state.go new file mode 100644 index 000000000..7bf2059e2 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/state.go @@ -0,0 +1,100 @@ +package v2 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +var log = logging.Logger("extract/actors/verifreg") + +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case VerifiersChangeList: + stateDiff.VerifierChanges = v + case ClaimsChangeMap: + stateDiff.ClaimChanges = v + case AllocationsChangeMap: + stateDiff.AllocationsChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) + } + } + return stateDiff, nil + +} + +type StateDiffResult struct { + VerifierChanges VerifiersChangeList + ClaimChanges ClaimsChangeMap + AllocationsChanges AllocationsChangeMap +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, store store.Store) (cbg.CBORMarshaler, error) { + out := &StateChange{} + + if verifiers := sd.VerifierChanges; verifiers != nil { + root, err := verifiers.ToAdtMap(store, 5) + if err != nil { + return nil, err + } + out.Verifiers = &root + } + + if claims := sd.ClaimChanges; claims != nil { + root, err := claims.ToAdtMap(store, 5) + if err != nil { + return nil, err + } + out.Claims = &root + } + + if allocations := sd.AllocationsChanges; allocations != nil { + root, err := allocations.ToAdtMap(store, 5) + if err != nil { + return nil, err + } + out.Allocations = &root + } + return out, nil +} + +type StateChange struct { + Verifiers *cid.Cid `cborgen:"verifiers"` + Claims *cid.Cid `cborgen:"claims"` + Allocations *cid.Cid `cborgen:"allocations"` +} + +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + VerifierChanges: VerifiersChangeList{}, + } + + if sc.Verifiers != nil { + verifierMap, err := adt.AsMap(s, *sc.Verifiers, 5) + if err != nil { + return nil, err + } + + verifierChange := new(VerifiersChange) + if err := verifierMap.ForEach(verifierChange, func(key string) error { + val := new(VerifiersChange) + *val = *verifierChange + out.VerifierChanges = append(out.VerifierChanges, val) + return nil + }); err != nil { + return nil, err + } + } + + return out, nil +} diff --git a/pkg/extract/actors/verifregdiff/v2/verifiers.go b/pkg/extract/actors/verifregdiff/v2/verifiers.go new file mode 100644 index 000000000..0cf2e372c --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/verifiers.go @@ -0,0 +1,9 @@ +package v2 + +import ( + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/version.go b/pkg/extract/actors/verifregdiff/version.go new file mode 100644 index 000000000..d1ba8c4e1 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/version.go @@ -0,0 +1,27 @@ +package verifregdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + v2 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + if av < actortypes.Version9 { + return []actors.ActorDiffMethods{ + v1.Clients{}, + v1.Verifiers{}, + }, v1.ActorStateChangeHandler, nil + } else if av == actortypes.Version9 { + return []actors.ActorDiffMethods{ + v2.Verifiers{}, + v2.Claims{}, + v2.Allocations{}, + }, v2.ActorStateChangeHandler, nil + } + return nil, nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/chain.go b/pkg/extract/chain.go new file mode 100644 index 000000000..d44b9b329 --- /dev/null +++ b/pkg/extract/chain.go @@ -0,0 +1,57 @@ +package extract + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/tasks" +) + +type MessageStateChanges struct { + BaseFee abi.TokenAmount + CirculatingSupply CirculatingSupply + FullBlocks map[cid.Cid]*chain.FullBlock + ImplicitMessages []*chain.ImplicitMessage +} + +type CirculatingSupply struct { + FilVested abi.TokenAmount + FilMined abi.TokenAmount + FilBurnt abi.TokenAmount + FilLocked abi.TokenAmount + FilCirculating abi.TokenAmount + FilReserveDisbursed abi.TokenAmount +} + +func FullBlockMessages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*MessageStateChanges, error) { + baseFee, err := api.ComputeBaseFee(ctx, executed) + if err != nil { + return nil, err + } + cs, err := api.CirculatingSupply(ctx, current) + if err != nil { + return nil, err + } + fullBlocks, implicitMessages, err := chain.Messages(ctx, api, current, executed) + if err != nil { + return nil, err + } + + return &MessageStateChanges{ + BaseFee: baseFee, + CirculatingSupply: CirculatingSupply{ + FilVested: cs.FilVested, + FilMined: cs.FilMined, + FilBurnt: cs.FilBurnt, + FilLocked: cs.FilLocked, + FilCirculating: cs.FilCirculating, + FilReserveDisbursed: cs.FilReserveDisbursed, + }, + FullBlocks: fullBlocks, + ImplicitMessages: implicitMessages, + }, nil +} diff --git a/pkg/extract/chain/cbor_gen.go b/pkg/extract/chain/cbor_gen.go new file mode 100644 index 000000000..46590c24e --- /dev/null +++ b/pkg/extract/chain/cbor_gen.go @@ -0,0 +1,1701 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package chain + +import ( + "fmt" + "io" + "math" + "sort" + + exitcode "github.com/filecoin-project/go-state-types/exitcode" + types "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *ChainMessageReceipt) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Receipt (types.MessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.GasOutputs (chain.MessageGasOutputs) (struct) + if len("gas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gas")); err != nil { + return err + } + + if err := t.GasOutputs.MarshalCBOR(cw); err != nil { + return err + } + + // t.ActorError (chain.ActorError) (struct) + if len("errors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"errors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("errors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("errors")); err != nil { + return err + } + + if err := t.ActorError.MarshalCBOR(cw); err != nil { + return err + } + + // t.Index (int64) (int64) + if len("index") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"index\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("index")); err != nil { + return err + } + + if t.Index >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Index)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Index-1)); err != nil { + return err + } + } + return nil +} + +func (t *ChainMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { + *t = ChainMessageReceipt{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ChainMessageReceipt: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Receipt (types.MessageReceipt) (struct) + case "receipt": + + { + + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt: %w", err) + } + + } + // t.GasOutputs (chain.MessageGasOutputs) (struct) + case "gas": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.GasOutputs = new(MessageGasOutputs) + if err := t.GasOutputs.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.GasOutputs pointer: %w", err) + } + } + + } + // t.ActorError (chain.ActorError) (struct) + case "errors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.ActorError = new(ActorError) + if err := t.ActorError.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.ActorError pointer: %w", err) + } + } + + } + // t.Index (int64) (int64) + case "index": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Index = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ImplicitMessageReceipt) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Receipt (types.MessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.GasOutputs (chain.MessageGasOutputs) (struct) + if len("gas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gas")); err != nil { + return err + } + + if err := t.GasOutputs.MarshalCBOR(cw); err != nil { + return err + } + + // t.ActorError (chain.ActorError) (struct) + if len("errors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"errors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("errors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("errors")); err != nil { + return err + } + + if err := t.ActorError.MarshalCBOR(cw); err != nil { + return err + } + return nil +} + +func (t *ImplicitMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { + *t = ImplicitMessageReceipt{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ImplicitMessageReceipt: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Receipt (types.MessageReceipt) (struct) + case "receipt": + + { + + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt: %w", err) + } + + } + // t.GasOutputs (chain.MessageGasOutputs) (struct) + case "gas": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.GasOutputs = new(MessageGasOutputs) + if err := t.GasOutputs.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.GasOutputs pointer: %w", err) + } + } + + } + // t.ActorError (chain.ActorError) (struct) + case "errors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.ActorError = new(ActorError) + if err := t.ActorError.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.ActorError pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *MessageGasOutputs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{167}); err != nil { + return err + } + + // t.BaseFeeBurn (big.Int) (struct) + if len("basefeeburn") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"basefeeburn\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("basefeeburn"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("basefeeburn")); err != nil { + return err + } + + if err := t.BaseFeeBurn.MarshalCBOR(cw); err != nil { + return err + } + + // t.OverEstimationBurn (big.Int) (struct) + if len("overestimationburn") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"overestimationburn\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("overestimationburn"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("overestimationburn")); err != nil { + return err + } + + if err := t.OverEstimationBurn.MarshalCBOR(cw); err != nil { + return err + } + + // t.MinerPenalty (big.Int) (struct) + if len("minerpenalty") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"minerpenalty\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("minerpenalty"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("minerpenalty")); err != nil { + return err + } + + if err := t.MinerPenalty.MarshalCBOR(cw); err != nil { + return err + } + + // t.MinerTip (big.Int) (struct) + if len("minertip") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"minertip\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("minertip"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("minertip")); err != nil { + return err + } + + if err := t.MinerTip.MarshalCBOR(cw); err != nil { + return err + } + + // t.Refund (big.Int) (struct) + if len("refund") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"refund\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("refund"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("refund")); err != nil { + return err + } + + if err := t.Refund.MarshalCBOR(cw); err != nil { + return err + } + + // t.GasRefund (int64) (int64) + if len("gasrufund") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gasrufund\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gasrufund"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gasrufund")); err != nil { + return err + } + + if t.GasRefund >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.GasRefund)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.GasRefund-1)); err != nil { + return err + } + } + + // t.GasBurned (int64) (int64) + if len("gasburned") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gasburned\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gasburned"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gasburned")); err != nil { + return err + } + + if t.GasBurned >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.GasBurned)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.GasBurned-1)); err != nil { + return err + } + } + return nil +} + +func (t *MessageGasOutputs) UnmarshalCBOR(r io.Reader) (err error) { + *t = MessageGasOutputs{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("MessageGasOutputs: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.BaseFeeBurn (big.Int) (struct) + case "basefeeburn": + + { + + if err := t.BaseFeeBurn.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BaseFeeBurn: %w", err) + } + + } + // t.OverEstimationBurn (big.Int) (struct) + case "overestimationburn": + + { + + if err := t.OverEstimationBurn.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.OverEstimationBurn: %w", err) + } + + } + // t.MinerPenalty (big.Int) (struct) + case "minerpenalty": + + { + + if err := t.MinerPenalty.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.MinerPenalty: %w", err) + } + + } + // t.MinerTip (big.Int) (struct) + case "minertip": + + { + + if err := t.MinerTip.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.MinerTip: %w", err) + } + + } + // t.Refund (big.Int) (struct) + case "refund": + + { + + if err := t.Refund.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Refund: %w", err) + } + + } + // t.GasRefund (int64) (int64) + case "gasrufund": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.GasRefund = int64(extraI) + } + // t.GasBurned (int64) (int64) + case "gasburned": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.GasBurned = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ActorError) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Fatal (bool) (bool) + if len("fatal") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"fatal\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fatal"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("fatal")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.Fatal); err != nil { + return err + } + + // t.RetCode (exitcode.ExitCode) (int64) + if len("retcode") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"retcode\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("retcode"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("retcode")); err != nil { + return err + } + + if t.RetCode >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.RetCode)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.RetCode-1)); err != nil { + return err + } + } + + // t.Error (string) (string) + if len("error") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"error\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("error"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("error")); err != nil { + return err + } + + if len(t.Error) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Error was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Error))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Error)); err != nil { + return err + } + return nil +} + +func (t *ActorError) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorError{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorError: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Fatal (bool) (bool) + case "fatal": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Fatal = false + case 21: + t.Fatal = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.RetCode (exitcode.ExitCode) (int64) + case "retcode": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.RetCode = exitcode.ExitCode(extraI) + } + // t.Error (string) (string) + case "error": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Error = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *VmMessage) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Source (cid.Cid) (struct) + if len("source") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"source\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("source"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("source")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Source); err != nil { + return xerrors.Errorf("failed to write cid field t.Source: %w", err) + } + + // t.Message (types.Message) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (types.MessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.Error (string) (string) + if len("error") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"error\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("error"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("error")); err != nil { + return err + } + + if len(t.Error) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Error was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Error))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Error)); err != nil { + return err + } + + // t.Index (int64) (int64) + if len("index") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"index\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("index")); err != nil { + return err + } + + if t.Index >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Index)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Index-1)); err != nil { + return err + } + } + return nil +} + +func (t *VmMessage) UnmarshalCBOR(r io.Reader) (err error) { + *t = VmMessage{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("VmMessage: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Source (cid.Cid) (struct) + case "source": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Source: %w", err) + } + + t.Source = c + + } + // t.Message (types.Message) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.Message) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (types.MessageReceipt) (struct) + case "receipt": + + { + + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt: %w", err) + } + + } + // t.Error (string) (string) + case "error": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Error = string(sval) + } + // t.Index (int64) (int64) + case "index": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Index = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *VmMessageGasTrace) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{168}); err != nil { + return err + } + + // t.Name (string) (string) + if len("name") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("name")); err != nil { + return err + } + + if len(t.Name) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Name)); err != nil { + return err + } + + // t.Location ([]chain.Loc) (slice) + if len("location") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"location\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("location"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("location")); err != nil { + return err + } + + if len(t.Location) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Location was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Location))); err != nil { + return err + } + for _, v := range t.Location { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.TotalGas (int64) (int64) + if len("totalgas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"totalgas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("totalgas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("totalgas")); err != nil { + return err + } + + if t.TotalGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TotalGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TotalGas-1)); err != nil { + return err + } + } + + // t.ComputeGas (int64) (int64) + if len("computegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"computegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("computegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("computegas")); err != nil { + return err + } + + if t.ComputeGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ComputeGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.ComputeGas-1)); err != nil { + return err + } + } + + // t.StorageGas (int64) (int64) + if len("storagegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"storagegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("storagegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("storagegas")); err != nil { + return err + } + + if t.StorageGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StorageGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StorageGas-1)); err != nil { + return err + } + } + + // t.TotalVirtualGas (int64) (int64) + if len("totalvirtgas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"totalvirtgas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("totalvirtgas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("totalvirtgas")); err != nil { + return err + } + + if t.TotalVirtualGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TotalVirtualGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TotalVirtualGas-1)); err != nil { + return err + } + } + + // t.VirtualComputeGas (int64) (int64) + if len("virtcomputegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"virtcomputegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("virtcomputegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("virtcomputegas")); err != nil { + return err + } + + if t.VirtualComputeGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.VirtualComputeGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.VirtualComputeGas-1)); err != nil { + return err + } + } + + // t.VirtualStorageGas (int64) (int64) + if len("cirtstoragegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"cirtstoragegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cirtstoragegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("cirtstoragegas")); err != nil { + return err + } + + if t.VirtualStorageGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.VirtualStorageGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.VirtualStorageGas-1)); err != nil { + return err + } + } + return nil +} + +func (t *VmMessageGasTrace) UnmarshalCBOR(r io.Reader) (err error) { + *t = VmMessageGasTrace{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("VmMessageGasTrace: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Name = string(sval) + } + // t.Location ([]chain.Loc) (slice) + case "location": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Location: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Location = make([]Loc, extra) + } + + for i := 0; i < int(extra); i++ { + + var v Loc + if err := v.UnmarshalCBOR(cr); err != nil { + return err + } + + t.Location[i] = v + } + + // t.TotalGas (int64) (int64) + case "totalgas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.TotalGas = int64(extraI) + } + // t.ComputeGas (int64) (int64) + case "computegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.ComputeGas = int64(extraI) + } + // t.StorageGas (int64) (int64) + case "storagegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.StorageGas = int64(extraI) + } + // t.TotalVirtualGas (int64) (int64) + case "totalvirtgas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.TotalVirtualGas = int64(extraI) + } + // t.VirtualComputeGas (int64) (int64) + case "virtcomputegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.VirtualComputeGas = int64(extraI) + } + // t.VirtualStorageGas (int64) (int64) + case "cirtstoragegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.VirtualStorageGas = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *Loc) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.File (string) (string) + if len("file") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"file\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("file"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("file")); err != nil { + return err + } + + if len(t.File) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.File was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.File))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.File)); err != nil { + return err + } + + // t.Line (int64) (int64) + if len("line") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"line\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("line"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("line")); err != nil { + return err + } + + if t.Line >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Line)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Line-1)); err != nil { + return err + } + } + + // t.Function (string) (string) + if len("function") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"function\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("function"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("function")); err != nil { + return err + } + + if len(t.Function) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Function was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Function))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Function)); err != nil { + return err + } + return nil +} + +func (t *Loc) UnmarshalCBOR(r io.Reader) (err error) { + *t = Loc{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Loc: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.File (string) (string) + case "file": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.File = string(sval) + } + // t.Line (int64) (int64) + case "line": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Line = int64(extraI) + } + // t.Function (string) (string) + case "function": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Function = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/chain/messages.go b/pkg/extract/chain/messages.go new file mode 100644 index 000000000..398d0a915 --- /dev/null +++ b/pkg/extract/chain/messages.go @@ -0,0 +1,276 @@ +package chain + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/lens" + "github.com/filecoin-project/lily/tasks" +) + +/* + type RootState struct { + // StateVersion contains the version of the State data structure. + // If changes are made to its structure, this version will increment. + StateVersion uint64 + + // State contains a single StateExtraction. + State cid.Cid + } + + type StateExtraction struct { + // Current and Parent are the TipSets whose state is diffed to produce Actors. + // BaseFee is calculated from Parent. + // FullBlocks contains the blocks and messages from Parent and receipts from Current. + // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. + // Actors contains actors whos state changed between Parent and Current. + + // Current contains the tipset whose state root is the result of Parent's execution. + Current *types.TipSet + // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. + Parent *types.TipSet + // BaseFee contains the basefee during Parent's execution. + BaseFee abi.TokenAmount + // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. + FullBlocks map[cid.Cid]*FullBlock + // ImplicitMessages contains a list of all implicit messages executed at Parent. + ImplicitMessages []*ImplicitMessage + // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. + Actors ActorStateChanges + } + + type StateExtractionIPLD struct { + // Current and Parent are the TipSets whose state is diffed to produce Actors. + // BaseFee is calculated from Parent. + // FullBlocks contains the blocks and messages from Parent and the messages respective receipts from Current. + // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. + // Actors contains actors whos state changed between Parent and Current. + + // Current contains the tipset whose state root is the result of Parent's execution. + Current *types.TipSet + // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. + Parent *types.TipSet + + // BaseFee contains the basefee during Parent's execution. + BaseFee abi.TokenAmount + + // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. + FullBlocks cid.Cid // HAMT[BlockHeaderCID]FullBlock + + // ImplicitMessages contains a map of all implicit messages executed at Parent. + ImplicitMessages cid.Cid // HAMT[MessageCID]ImplicitMessage + + // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. + Actors cid.Cid // ActorStateChanges + } +*/ + +type FullBlock struct { + Block *types.BlockHeader + SecpMessages []*SignedChainMessage + BlsMessages []*ChainMessage +} + +// SignedChainMessage is a signed (secp) message appearing on chain. Receipt is null if the message was not executed. +type SignedChainMessage struct { + Message *types.SignedMessage + Receipt *ChainMessageReceipt + VmMessages VmMessageList +} + +// ChainMessage is an unsigned (bls) message appearing on chain. Receipt is null if the message was not executed. +type ChainMessage struct { + Message *types.Message + Receipt *ChainMessageReceipt + VmMessages VmMessageList +} + +// ImplicitMessage is an implicitly executed message not appearing on chain. +type ImplicitMessage struct { + Message *types.Message + Receipt *ImplicitMessageReceipt + VmMessages VmMessageList +} + +// ChainMessageReceipt contains a MessageReceipt and other metadata. +type ChainMessageReceipt struct { + Receipt types.MessageReceipt `cborgen:"receipt"` + GasOutputs *MessageGasOutputs `cborgen:"gas"` + ActorError *ActorError `cborgen:"errors"` + Index int64 `cborgen:"index"` +} + +type ImplicitMessageReceipt struct { + Receipt types.MessageReceipt `cborgen:"receipt"` + GasOutputs *MessageGasOutputs `cborgen:"gas"` + ActorError *ActorError `cborgen:"errors"` +} + +// MessageGasOutputs contains the gas used during a message's execution. +type MessageGasOutputs struct { + BaseFeeBurn abi.TokenAmount `cborgen:"basefeeburn"` + OverEstimationBurn abi.TokenAmount `cborgen:"overestimationburn"` + MinerPenalty abi.TokenAmount `cborgen:"minerpenalty"` + MinerTip abi.TokenAmount `cborgen:"minertip"` + Refund abi.TokenAmount `cborgen:"refund"` + GasRefund int64 `cborgen:"gasrufund"` + GasBurned int64 `cborgen:"gasburned"` +} + +// ActorError contains any errors encountered during a message's execution. +type ActorError struct { + Fatal bool `cborgen:"fatal"` + RetCode exitcode.ExitCode `cborgen:"retcode"` + Error string `cborgen:"error"` +} + +func Messages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (map[cid.Cid]*FullBlock, []*ImplicitMessage, error) { + var ( + blkMsgs []*lens.BlockMessages + exeBlkMsgs []*lens.BlockMessageReceipts + msgExe []*lens.MessageExecutionV2 + err error + ) + grp, grpCtx := errgroup.WithContext(ctx) + + grp.Go(func() error { + blkMsgs, err = api.TipSetBlockMessages(grpCtx, executed) + if err != nil { + return err + } + return nil + }) + + grp.Go(func() error { + exeBlkMsgs, err = api.TipSetMessageReceipts(grpCtx, current, executed) + if err != nil { + return err + } + return nil + }) + + grp.Go(func() error { + msgExe, err = api.MessageExecutionsV2(grpCtx, current, executed) + if err != nil { + return err + } + return nil + }) + if err := grp.Wait(); err != nil { + return nil, nil, err + } + + // build two maps containing all signed (secpMsgs) and unsigned (blsMsgs) messages. + secpBlkMsgs := make(map[cid.Cid][]*SignedChainMessage) + blsBlkMsgs := make(map[cid.Cid][]*ChainMessage) + secpMsgs := make(map[cid.Cid]*SignedChainMessage) + blsMsgs := make(map[cid.Cid]*ChainMessage) + for _, blk := range blkMsgs { + blkCid := blk.Block.Cid() + for _, msg := range blk.SecpMessages { + secpMsg := &SignedChainMessage{Message: msg} + secpBlkMsgs[blkCid] = append(secpBlkMsgs[blkCid], secpMsg) + secpMsgs[msg.Cid()] = secpMsg + } + for _, msg := range blk.BlsMessages { + blsMsg := &ChainMessage{Message: msg} + blsBlkMsgs[blkCid] = append(blsBlkMsgs[blkCid], blsMsg) + blsMsgs[msg.Cid()] = blsMsg + } + } + + for _, ebm := range exeBlkMsgs { + itr, err := ebm.Iterator() + if err != nil { + return nil, nil, err + } + for itr.HasNext() { + msg, recIdx, rec := itr.Next() + if secpMsg, ok := secpMsgs[msg.Cid()]; ok { + secpMsg.Receipt = &ChainMessageReceipt{ + Receipt: *rec, + Index: int64(recIdx), + } + } else if blsMsg, ok := blsMsgs[msg.Cid()]; ok { + blsMsg.Receipt = &ChainMessageReceipt{ + Receipt: *rec, + Index: int64(recIdx), + } + } else { + panic("developer error") + } + } + } + + var im []*ImplicitMessage + for _, emsg := range msgExe { + vmMsgs, err := ProcessVmMessages(ctx, emsg) + if err != nil { + return nil, nil, err + } + if emsg.Implicit { + im = append(im, &ImplicitMessage{ + Message: emsg.Message, + VmMessages: vmMsgs, + Receipt: &ImplicitMessageReceipt{ + Receipt: emsg.Ret.MessageReceipt, + GasOutputs: GetMessageGasOutputs(emsg), + ActorError: GetActorError(emsg), + }, + }) + } else { + if secpMsg, ok := secpMsgs[emsg.Cid]; ok { + secpMsg.Receipt.GasOutputs = GetMessageGasOutputs(emsg) + secpMsg.Receipt.ActorError = GetActorError(emsg) + secpMsg.VmMessages = vmMsgs + } else if blsMsg, ok := blsMsgs[emsg.Cid]; ok { + blsMsg.Receipt.ActorError = GetActorError(emsg) + blsMsg.Receipt.GasOutputs = GetMessageGasOutputs(emsg) + blsMsg.VmMessages = vmMsgs + } else { + panic("developer error") + } + } + } + + fullBlocks := make(map[cid.Cid]*FullBlock) + for _, blk := range blkMsgs { + fullBlocks[blk.Block.Cid()] = &FullBlock{ + Block: blk.Block, + SecpMessages: secpBlkMsgs[blk.Block.Cid()], + BlsMessages: blsBlkMsgs[blk.Block.Cid()], + } + } + return fullBlocks, im, nil +} + +func GetMessageGasOutputs(msg *lens.MessageExecutionV2) *MessageGasOutputs { + if msg.Ret.GasCosts != nil { + return &MessageGasOutputs{ + BaseFeeBurn: msg.Ret.GasCosts.BaseFeeBurn, + OverEstimationBurn: msg.Ret.GasCosts.OverEstimationBurn, + MinerPenalty: msg.Ret.GasCosts.MinerPenalty, + MinerTip: msg.Ret.GasCosts.MinerTip, + Refund: msg.Ret.GasCosts.Refund, + GasRefund: msg.Ret.GasCosts.GasRefund, + GasBurned: msg.Ret.GasCosts.GasBurned, + } + } + return nil +} + +func GetActorError(msg *lens.MessageExecutionV2) *ActorError { + if msg.Ret.ActorErr != nil { + return &ActorError{ + Fatal: msg.Ret.ActorErr.IsFatal(), + RetCode: msg.Ret.ActorErr.RetCode(), + Error: msg.Ret.ActorErr.Error(), + } + } + return nil +} diff --git a/pkg/extract/chain/vm.go b/pkg/extract/chain/vm.go new file mode 100644 index 000000000..54815fa5e --- /dev/null +++ b/pkg/extract/chain/vm.go @@ -0,0 +1,149 @@ +package chain + +import ( + "context" + "time" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/lens" +) + +type VmMessage struct { + Source cid.Cid `cborgen:"source"` + Message *types.Message `cborgen:"message"` + Receipt types.MessageReceipt `cborgen:"receipt"` + Error string `cborgen:"error"` + Index int64 `cborgen:"index"` +} + +type VmMessageGasTrace struct { + Name string `cborgen:"name"` + Location []Loc `cborgen:"location"` + TotalGas int64 `cborgen:"totalgas"` + ComputeGas int64 `cborgen:"computegas"` + StorageGas int64 `cborgen:"storagegas"` + TotalVirtualGas int64 `cborgen:"totalvirtgas"` + VirtualComputeGas int64 `cborgen:"virtcomputegas"` + VirtualStorageGas int64 `cborgen:"cirtstoragegas"` +} + +type Loc struct { + File string `cborgen:"file"` + Line int64 `cborgen:"line"` + Function string `cborgen:"function"` +} + +type VmMessageList []*VmMessage + +func (vml VmMessageList) ToAdtArray(store adt.Store, bw int) (cid.Cid, error) { + msgAmt, err := adt.MakeEmptyArray(store, bw) + if err != nil { + return cid.Undef, nil + } + + for _, msg := range vml { + if err := msgAmt.Set(uint64(msg.Index), msg); err != nil { + return cid.Undef, err + } + } + return msgAmt.Root() +} + +func VmMessageListFromAdtArray(store adt.Store, root cid.Cid, bw int) (VmMessageList, error) { + arr, err := adt.AsArray(store, root, bw) + if err != nil { + return nil, err + } + out := make(VmMessageList, arr.Length()) + msg := new(VmMessage) + idx := 0 + if err := arr.ForEach(msg, func(i int64) error { + val := new(VmMessage) + *val = *msg + out[idx] = val + idx++ + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +func ProcessVmMessages(ctx context.Context, source *lens.MessageExecutionV2) (VmMessageList, error) { + trace := GetChildMessagesOf(source) + out := make([]*VmMessage, len(trace)) + for traceIdx, vmmsg := range trace { + // see TODO on VmMessage struct + /* + vmGas := make([]*VmMessageGasTrace, len(vmmsg.GasCharge)) + + for gasIdx, g := range vmmsg.GasCharge { + loc := make([]Loc, len(g.Location)) + + for locIdx, l := range g.Location { + loc[locIdx] = Loc{ + File: l.File, + Line: int64(l.Line), + Function: l.Function, + } + } + + vmGas[gasIdx] = &VmMessageGasTrace{ + Name: g.Name, + Location: loc, + TotalGas: g.TotalGas, + ComputeGas: g.ComputeGas, + StorageGas: g.StorageGas, + TotalVirtualGas: g.TotalVirtualGas, + VirtualComputeGas: g.VirtualComputeGas, + VirtualStorageGas: g.VirtualStorageGas, + } + } + + */ + + out[traceIdx] = &VmMessage{ + Source: source.Cid, + Message: vmmsg.Message, + Receipt: *vmmsg.Receipt, + //GasTrace: vmGas, + Error: vmmsg.Error, + Index: int64(vmmsg.Index), + } + } + return out, nil +} + +type vmMessageTrace struct { + Message *types.Message + Receipt *types.MessageReceipt + Error string + Duration time.Duration + GasCharge []*types.GasTrace + Index int +} + +func GetChildMessagesOf(m *lens.MessageExecutionV2) []*vmMessageTrace { + var out []*vmMessageTrace + index := 0 + walkExecutionTrace(&m.Ret.ExecutionTrace, &out, &index) + return out +} + +func walkExecutionTrace(et *types.ExecutionTrace, trace *[]*vmMessageTrace, index *int) { + for _, sub := range et.Subcalls { + *trace = append(*trace, &vmMessageTrace{ + Message: sub.Msg, + Receipt: sub.MsgRct, + Error: sub.Error, + Duration: sub.Duration, + GasCharge: sub.GasCharges, + Index: *index, + }) + *index++ + walkExecutionTrace(&sub, trace, index) //nolint:scopelint,gosec + } +} diff --git a/pkg/extract/cleanup.go b/pkg/extract/cleanup.go new file mode 100644 index 000000000..7e3d3b908 --- /dev/null +++ b/pkg/extract/cleanup.go @@ -0,0 +1 @@ +package extract diff --git a/pkg/extract/extract.go b/pkg/extract/extract.go new file mode 100644 index 000000000..e0a2fc0db --- /dev/null +++ b/pkg/extract/extract.go @@ -0,0 +1,85 @@ +package extract + +import ( + "context" + "runtime" + "time" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/types" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/tasks" +) + +type ChainState struct { + NetworkName string + NetworkVersion uint64 + ActorVersion uint64 + Current *types.TipSet + Parent *types.TipSet + Message *MessageStateChanges + Actors *ActorStateChanges +} + +func State(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*ChainState, error) { + var ( + blockmessages *MessageStateChanges + actorChanges *ActorStateChanges + err error + ) + + networkName, err := api.NetworkName(ctx) + if err != nil { + return nil, err + } + + networkVersion, err := api.NetworkVersion(ctx, current.Key()) + if err != nil { + return nil, err + } + + actorVersion, err := actortypes.VersionForNetwork(network.Version(networkVersion)) + if err != nil { + return nil, err + } + + grp, grpCtx := errgroup.WithContext(ctx) + grp.Go(func() error { + start := time.Now() + // all blocks, messages, implicit messages, from executed and receipts from current + blockmessages, err = FullBlockMessages(grpCtx, api, current, executed) + if err != nil { + log.Errorw("failed to extract full block messages", "error", err) + return err + } + log.Infow("extracted full block messages", "duration", time.Since(start)) + return nil + }) + grp.Go(func() error { + start := time.Now() + // all actor changes between current and parent, actor state exists in current. + actorChanges, err = Actors(ctx, api, current, executed, runtime.NumCPU()) + if err != nil { + log.Errorw("failed to extract actor states", "error", err) + return err + } + log.Infow("extracted actor states", "duration", time.Since(start)) + return nil + }) + + if err := grp.Wait(); err != nil { + return nil, err + } + + return &ChainState{ + NetworkName: networkName, + NetworkVersion: uint64(networkVersion), + ActorVersion: uint64(actorVersion), + Current: current, + Parent: executed, + Message: blockmessages, + Actors: actorChanges, + }, nil +} diff --git a/pkg/extract/statetree/diff.go b/pkg/extract/statetree/diff.go new file mode 100644 index 000000000..18b0109b9 --- /dev/null +++ b/pkg/extract/statetree/diff.go @@ -0,0 +1,188 @@ +package statetree + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin" + "github.com/filecoin-project/lotus/chain/types" + states0 "github.com/filecoin-project/specs-actors/actors/states" + states2 "github.com/filecoin-project/specs-actors/v2/actors/states" + states3 "github.com/filecoin-project/specs-actors/v3/actors/states" + states4 "github.com/filecoin-project/specs-actors/v4/actors/states" + states5 "github.com/filecoin-project/specs-actors/v5/actors/states" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/pkg/core" +) + +var log = logging.Logger("lily/extract/statetree") + +type ActorDiff struct { + Executed *types.Actor + Current *types.Actor + ChangeType core.ChangeType +} + +func DiffActorStateTree(ctx context.Context, store adt.Store, current, executed *types.TipSet) (core.MapModifications, error) { + // we have this special method here to get the HAMT node root required by the faster diffing logic. I hate this. + executedMap, executedMapOps, err := getStateTreeHamtRootCIDAndVersion(ctx, store, executed.ParentState()) + if err != nil { + return nil, err + } + currentMap, currentMapOps, err := getStateTreeHamtRootCIDAndVersion(ctx, store, current.ParentState()) + if err != nil { + return nil, err + } + + changes, err := core.DiffMap(ctx, store, currentMap, executedMap, currentMapOps, executedMapOps) + if err != nil { + return nil, err + } + return changes, nil +} + +func ActorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet) (map[address.Address]ActorDiff, error) { + start := time.Now() + defer func() { + log.Infow("Actor Changes", "duration", time.Since(start)) + }() + changes, err := DiffActorStateTree(ctx, store, current, executed) + if err != nil { + return nil, err + } + buf := bytes.NewReader(nil) + out := map[address.Address]ActorDiff{} + for _, change := range changes { + addr, err := address.NewFromBytes(change.Key) + if err != nil { + return nil, fmt.Errorf("address in state tree was not valid: %w", err) + } + ch := ActorDiff{ + Executed: new(types.Actor), + Current: new(types.Actor), + ChangeType: core.ChangeTypeUnknown, + } + switch change.Type { + case core.ChangeTypeAdd: + ch.ChangeType = change.Type + buf.Reset(change.Current.Raw) + err = ch.Current.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + + case core.ChangeTypeRemove: + ch.ChangeType = change.Type + buf.Reset(change.Previous.Raw) + err = ch.Executed.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + + case core.ChangeTypeModify: + ch.ChangeType = change.Type + buf.Reset(change.Previous.Raw) + err = ch.Executed.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + + buf.Reset(change.Current.Raw) + err = ch.Current.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + } + out[addr] = ch + } + return out, nil +} + +func getStateTreeHamtRootCIDAndVersion(ctx context.Context, store adt.Store, c cid.Cid) (adt.Map, *adt.MapOpts, error) { + var root types.StateRoot + // Try loading as a new-style state-tree (version/actors tuple). + if err := store.Get(ctx, c, &root); err != nil { + // We failed to decode as the new version, must be an old version. + root.Actors = c + root.Version = types.StateTreeVersion0 + } + + switch root.Version { + case types.StateTreeVersion0: + var tree *states0.Tree + tree, err := states0.LoadTree(store, root.Actors) + if err != nil { + return nil, nil, err + } + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + + case types.StateTreeVersion1: + var tree *states2.Tree + tree, err := states2.LoadTree(store, root.Actors) + if err != nil { + return nil, nil, err + } + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + + case types.StateTreeVersion2: + var tree *states3.Tree + tree, err := states3.LoadTree(store, root.Actors) + if err != nil { + return nil, nil, err + } + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + + case types.StateTreeVersion3: + var tree *states4.Tree + tree, err := states4.LoadTree(store, root.Actors) + if err != nil { + return nil, nil, err + } + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + case types.StateTreeVersion4: + var tree *states5.Tree + tree, err := states5.LoadTree(store, root.Actors) + if err != nil { + return nil, nil, err + } + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + default: + return nil, nil, fmt.Errorf("unsupported state tree version: %d", root.Version) + } +} diff --git a/pkg/gen/main.go b/pkg/gen/main.go new file mode 100644 index 000000000..240978887 --- /dev/null +++ b/pkg/gen/main.go @@ -0,0 +1,152 @@ +package main + +import ( + cbg "github.com/whyrusleeping/cbor-gen" + + datacapV9 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v1" + initv0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" + marketV0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + minerV0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" + verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" +) + +const actorDiffPath = "pkg/extract/actors/rawdiff/cbor_gen.go" +const actorDiffPkg = "rawdiff" + +const datacapDiffPath = "pkg/extract/actors/datacapdiff/v1/cbor_gen.go" +const datacapDiffPkg = "v1" + +const minerDiffPath = "pkg/extract/actors/minerdiff/v1/cbor_gen.go" +const minerDiffPkg = "v1" + +const initDiffPath = "pkg/extract/actors/initdiff/v1/cbor_gen.go" +const initDiffPkg = "v1" + +const verifDiffPathV0 = "pkg/extract/actors/verifregdiff/v1/cbor_gen.go" +const verifDiffPkgV0 = "v1" + +const verifDiffPathV9 = "pkg/extract/actors/verifregdiff/v2/cbor_gen.go" +const verifDiffPkgV9 = "v2" + +const marketDiffPath = "pkg/extract/actors/marketdiff/v1/cbor_gen.go" +const marketDiffPkg = "v1" + +const powerDiffPath = "pkg/extract/actors/powerdiff/v1/cbor_gen.go" +const powerDiffPkg = "v1" + +const IPLDActorContainerPath = "pkg/transform/cbor/actors/cbor_gen.go" +const IPLDActorContainerPkg = "actors" + +const MessageStatePath = "pkg/extract/chain/cbor_gen.go" +const MessageStatePkg = "chain" + +const MessageContainerPath = "pkg/transform/cbor/messages/cbor_gen.go" +const MessageContainerPkg = "messages" + +const RootStatePath = "pkg/transform/cbor/cbor_gen.go" +const RootStatePkg = "cbor" + +func main() { + if err := cbg.WriteMapEncodersToFile(datacapDiffPath, datacapDiffPkg, + datacapV9.AllowanceChange{}, + datacapV9.BalanceChange{}, + datacapV9.StateChange{}, + ); err != nil { + panic(err) + } + if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, + rawdiff.ActorChange{}, + rawdiff.StateChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(minerDiffPath, minerDiffPkg, + minerV0.SectorStatusChange{}, + minerV0.PreCommitChange{}, + minerV0.SectorChange{}, + minerV0.InfoChange{}, + minerV0.StateChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(initDiffPath, initDiffPkg, + initv0.AddressChange{}, + initv0.StateChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(verifDiffPathV0, verifDiffPkgV0, + verifV0.StateChange{}, + verifV0.ClientsChange{}, + verifV0.VerifiersChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(verifDiffPathV9, verifDiffPkgV9, + verifV9.StateChange{}, + verifV9.ClaimsChange{}, + verifV9.AllocationsChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(marketDiffPath, marketDiffPkg, + marketV0.StateChange{}, + marketV0.ProposalChange{}, + marketV0.DealChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(powerDiffPath, powerDiffPkg, + powerV0.StateChange{}, + powerV0.ClaimsChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(IPLDActorContainerPath, IPLDActorContainerPkg, + actors.ActorStateChangesIPLD{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(MessageStatePath, MessageStatePkg, + chain.ChainMessageReceipt{}, + chain.ImplicitMessageReceipt{}, + chain.MessageGasOutputs{}, + chain.ActorError{}, + chain.VmMessage{}, + chain.VmMessageGasTrace{}, + chain.Loc{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(MessageContainerPath, MessageContainerPkg, + messages.FullBlockIPLDContainer{}, + messages.ChainMessageIPLDContainer{}, + messages.SignedChainMessageIPLDContainer{}, + messages.ImplicitMessageIPLDContainer{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(RootStatePath, RootStatePkg, + cbor.RootStateIPLD{}, + cbor.StateExtractionIPLD{}, + ); err != nil { + panic(err) + } +} diff --git a/pkg/indexer/inderxer.go b/pkg/indexer/inderxer.go new file mode 100644 index 000000000..e46379b5d --- /dev/null +++ b/pkg/indexer/inderxer.go @@ -0,0 +1,44 @@ +package indexer + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer" + "github.com/filecoin-project/lily/pkg/extract" + "github.com/filecoin-project/lily/tasks" +) + +func NewStateIndexer(api tasks.DataSource, handler StateHandler) *StateIndexer { + return &StateIndexer{ + api: api, + handler: handler, + } +} + +type StateIndexer struct { + api tasks.DataSource + handler StateHandler +} + +type StateHandler interface { + Persist(ctx context.Context, chainState *extract.ChainState) error +} + +func (s *StateIndexer) TipSet(ctx context.Context, ts *types.TipSet, opts ...indexer.Option) (bool, error) { + parent, err := s.api.TipSet(ctx, ts.Parents()) + if err != nil { + return false, err + } + + chainState, err := extract.State(ctx, s.api, ts, parent) + if err != nil { + return false, err + } + + if err := s.handler.Persist(ctx, chainState); err != nil { + return false, err + } + return true, nil +} diff --git a/pkg/transform/cbor/actors/cbor_gen.go b/pkg/transform/cbor/actors/cbor_gen.go new file mode 100644 index 000000000..f686fbb89 --- /dev/null +++ b/pkg/transform/cbor/actors/cbor_gen.go @@ -0,0 +1,397 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package actors + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{167}); err != nil { + return err + } + + // t.DataCapActor (cid.Cid) (struct) + if len("datacap") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"datacap\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("datacap"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("datacap")); err != nil { + return err + } + + if t.DataCapActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.DataCapActor); err != nil { + return xerrors.Errorf("failed to write cid field t.DataCapActor: %w", err) + } + } + + // t.InitActor (cid.Cid) (struct) + if len("init") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"init\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("init"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("init")); err != nil { + return err + } + + if t.InitActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.InitActor); err != nil { + return xerrors.Errorf("failed to write cid field t.InitActor: %w", err) + } + } + + // t.MarketActor (cid.Cid) (struct) + if len("market") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"market\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("market"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("market")); err != nil { + return err + } + + if t.MarketActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.MarketActor); err != nil { + return xerrors.Errorf("failed to write cid field t.MarketActor: %w", err) + } + } + + // t.MinerActors (cid.Cid) (struct) + if len("miner") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"miner\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miner"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("miner")); err != nil { + return err + } + + if t.MinerActors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.MinerActors); err != nil { + return xerrors.Errorf("failed to write cid field t.MinerActors: %w", err) + } + } + + // t.PowerActor (cid.Cid) (struct) + if len("power") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"power\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("power"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("power")); err != nil { + return err + } + + if t.PowerActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.PowerActor); err != nil { + return xerrors.Errorf("failed to write cid field t.PowerActor: %w", err) + } + } + + // t.RawActors (cid.Cid) (struct) + if len("raw") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"raw\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("raw"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("raw")); err != nil { + return err + } + + if t.RawActors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.RawActors); err != nil { + return xerrors.Errorf("failed to write cid field t.RawActors: %w", err) + } + } + + // t.VerifregActor (cid.Cid) (struct) + if len("verifreg") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifreg\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifreg"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifreg")); err != nil { + return err + } + + if t.VerifregActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.VerifregActor); err != nil { + return xerrors.Errorf("failed to write cid field t.VerifregActor: %w", err) + } + } + + return nil +} + +func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorStateChangesIPLD{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorStateChangesIPLD: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.DataCapActor (cid.Cid) (struct) + case "datacap": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.DataCapActor: %w", err) + } + + t.DataCapActor = &c + } + + } + // t.InitActor (cid.Cid) (struct) + case "init": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.InitActor: %w", err) + } + + t.InitActor = &c + } + + } + // t.MarketActor (cid.Cid) (struct) + case "market": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.MarketActor: %w", err) + } + + t.MarketActor = &c + } + + } + // t.MinerActors (cid.Cid) (struct) + case "miner": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.MinerActors: %w", err) + } + + t.MinerActors = &c + } + + } + // t.PowerActor (cid.Cid) (struct) + case "power": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PowerActor: %w", err) + } + + t.PowerActor = &c + } + + } + // t.RawActors (cid.Cid) (struct) + case "raw": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.RawActors: %w", err) + } + + t.RawActors = &c + } + + } + // t.VerifregActor (cid.Cid) (struct) + case "verifreg": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VerifregActor: %w", err) + } + + t.VerifregActor = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go new file mode 100644 index 000000000..26b39e2b9 --- /dev/null +++ b/pkg/transform/cbor/actors/serialize.go @@ -0,0 +1,187 @@ +package actors + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" + "gorm.io/gorm" + + "github.com/filecoin-project/lily/pkg/extract" + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +// TODO version the content at these CIDs +type ActorStateChangesIPLD struct { + DataCapActor *cid.Cid `cborgen:"datacap"` // DataCap + InitActor *cid.Cid `cborgen:"init"` // Init + MarketActor *cid.Cid `cborgen:"market"` // Market + MinerActors *cid.Cid `cborgen:"miner"` // HAMT[address]Miner + PowerActor *cid.Cid `cborgen:"power"` // Power + RawActors *cid.Cid `cborgen:"raw"` // HAMT[address]Raw + VerifregActor *cid.Cid `cborgen:"verifreg"` // Veriferg +} + +func (a *ActorStateChangesIPLD) AsModel() *ActorStateModel { + out := &ActorStateModel{} + if a.DataCapActor != nil { + out.DataCapActor = a.DataCapActor.String() + } + if a.InitActor != nil { + out.InitActor = a.InitActor.String() + } + if a.MarketActor != nil { + out.MarketActor = a.MarketActor.String() + } + if a.MinerActors != nil { + out.MinerActors = a.MinerActors.String() + } + if a.PowerActor != nil { + out.PowerActor = a.PowerActor.String() + } + if a.RawActors != nil { + out.RawActors = a.RawActors.String() + } + if a.VerifregActor != nil { + out.VerifregActor = a.VerifregActor.String() + } + return out +} + +type ActorStateModel struct { + gorm.Model + Height uint64 + DataCapActor string + InitActor string + MarketActor string + MinerActors string + PowerActor string + RawActors string + VerifregActor string +} + +func (a *ActorStateChangesIPLD) Attributes() []attribute.KeyValue { + var out []attribute.KeyValue + if a.DataCapActor != nil { + out = append(out, attribute.String("data_cap_root", a.DataCapActor.String())) + } + if a.InitActor != nil { + out = append(out, attribute.String("init_root", a.InitActor.String())) + } + if a.MarketActor != nil { + out = append(out, attribute.String("market_root", a.MarketActor.String())) + } + if a.MinerActors != nil { + out = append(out, attribute.String("miner_root", a.MinerActors.String())) + } + if a.PowerActor != nil { + out = append(out, attribute.String("power_root", a.PowerActor.String())) + } + if a.RawActors != nil { + out = append(out, attribute.String("raw_root", a.RawActors.String())) + } + if a.VerifregActor != nil { + out = append(out, attribute.String("verifreg_root", a.VerifregActor.String())) + } + return out +} + +func (a *ActorStateChangesIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range a.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + +func ProcessActorsStates(ctx context.Context, s store.Store, changes *extract.ActorStateChanges) (*ActorStateChangesIPLD, error) { + out := &ActorStateChangesIPLD{} + + if changes.DatacapActor != nil { + dcapRoot, err := PutActorDiffResult(ctx, s, changes.DatacapActor) + if err != nil { + return nil, err + } + out.DataCapActor = &dcapRoot + } + + if changes.InitActor != nil { + initRoot, err := PutActorDiffResult(ctx, s, changes.InitActor) + if err != nil { + return nil, err + } + out.InitActor = &initRoot + } + + if changes.MarketActor != nil { + marketRoot, err := PutActorDiffResult(ctx, s, changes.MarketActor) + if err != nil { + return nil, err + } + out.MarketActor = &marketRoot + } + + if changes.MinerActors != nil { + minerRoot, err := PutActorDiffResultMap(ctx, s, changes.MinerActors) + if err != nil { + return nil, err + } + out.MinerActors = &minerRoot + } + + if changes.PowerActor != nil { + powerRoot, err := PutActorDiffResult(ctx, s, changes.PowerActor) + if err != nil { + return nil, err + } + out.PowerActor = &powerRoot + } + + if changes.RawActors != nil { + actorsRoot, err := PutActorDiffResultMap(ctx, s, changes.RawActors) + if err != nil { + return nil, err + } + out.RawActors = &actorsRoot + } + + if changes.VerifregActor != nil { + verifregRoot, err := PutActorDiffResult(ctx, s, changes.VerifregActor) + if err != nil { + return nil, err + } + out.VerifregActor = &verifregRoot + } + + return out, nil +} + +func PutActorDiffResult(ctx context.Context, s store.Store, result actors.DiffResult) (cid.Cid, error) { + changes, err := result.MarshalStateChange(ctx, s) + if err != nil { + return cid.Undef, err + } + return s.Put(ctx, changes) +} + +func PutActorDiffResultMap(ctx context.Context, s store.Store, results map[address.Address]actors.DiffResult) (cid.Cid, error) { + actorHamt, err := adt.MakeEmptyMap(s, 5 /*TODO*/) + if err != nil { + return cid.Undef, err + } + for addr, change := range results { + msc, err := change.MarshalStateChange(ctx, s) + if err != nil { + return cid.Undef, err + } + + if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { + return cid.Undef, err + } + } + return actorHamt.Root() +} diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/cbor_gen.go new file mode 100644 index 000000000..1b81ca138 --- /dev/null +++ b/pkg/transform/cbor/cbor_gen.go @@ -0,0 +1,590 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package cbor + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *RootStateIPLD) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.StateVersion (uint64) (uint64) + if len("stateversion") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"stateversion\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("stateversion"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("stateversion")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StateVersion)); err != nil { + return err + } + + // t.NetworkName (string) (string) + if len("networkname") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"networkname\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("networkname"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("networkname")); err != nil { + return err + } + + if len(t.NetworkName) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.NetworkName was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.NetworkName))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.NetworkName)); err != nil { + return err + } + + // t.NetworkVersion (uint64) (uint64) + if len("networkversion") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"networkversion\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("networkversion"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("networkversion")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.NetworkVersion)); err != nil { + return err + } + + // t.State (cid.Cid) (struct) + if len("state") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"state\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("state"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("state")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.State); err != nil { + return xerrors.Errorf("failed to write cid field t.State: %w", err) + } + + return nil +} + +func (t *RootStateIPLD) UnmarshalCBOR(r io.Reader) (err error) { + *t = RootStateIPLD{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("RootStateIPLD: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.StateVersion (uint64) (uint64) + case "stateversion": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.StateVersion = uint64(extra) + + } + // t.NetworkName (string) (string) + case "networkname": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.NetworkName = string(sval) + } + // t.NetworkVersion (uint64) (uint64) + case "networkversion": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.NetworkVersion = uint64(extra) + + } + // t.State (cid.Cid) (struct) + case "state": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.State: %w", err) + } + + t.State = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateExtractionIPLD) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{172}); err != nil { + return err + } + + // t.Current (types.TipSet) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Parent (types.TipSet) (struct) + if len("parent") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"parent\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("parent"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("parent")); err != nil { + return err + } + + if err := t.Parent.MarshalCBOR(cw); err != nil { + return err + } + + // t.BaseFee (big.Int) (struct) + if len("basefee") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"basefee\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("basefee"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("basefee")); err != nil { + return err + } + + if err := t.BaseFee.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilVested (big.Int) (struct) + if len("filvested") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filvested\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filvested"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filvested")); err != nil { + return err + } + + if err := t.FilVested.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilMined (big.Int) (struct) + if len("filmined") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filmined\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filmined"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filmined")); err != nil { + return err + } + + if err := t.FilMined.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilBurnt (big.Int) (struct) + if len("filburnt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filburnt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filburnt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filburnt")); err != nil { + return err + } + + if err := t.FilBurnt.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilLocked (big.Int) (struct) + if len("fillocked") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"fillocked\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fillocked"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("fillocked")); err != nil { + return err + } + + if err := t.FilLocked.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilCirculating (big.Int) (struct) + if len("filcirculating") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filcirculating\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filcirculating"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filcirculating")); err != nil { + return err + } + + if err := t.FilCirculating.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilReserveDisbursed (big.Int) (struct) + if len("filreserveddisbursed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filreserveddisbursed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filreserveddisbursed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filreserveddisbursed")); err != nil { + return err + } + + if err := t.FilReserveDisbursed.MarshalCBOR(cw); err != nil { + return err + } + + // t.FullBlocks (cid.Cid) (struct) + if len("fullblocks") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"fullblocks\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fullblocks"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("fullblocks")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.FullBlocks); err != nil { + return xerrors.Errorf("failed to write cid field t.FullBlocks: %w", err) + } + + // t.ImplicitMessages (cid.Cid) (struct) + if len("implicitmessages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"implicitmessages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("implicitmessages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("implicitmessages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.ImplicitMessages); err != nil { + return xerrors.Errorf("failed to write cid field t.ImplicitMessages: %w", err) + } + + // t.Actors (cid.Cid) (struct) + if len("actors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"actors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("actors")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Actors); err != nil { + return xerrors.Errorf("failed to write cid field t.Actors: %w", err) + } + + return nil +} + +func (t *StateExtractionIPLD) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateExtractionIPLD{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateExtractionIPLD: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Current (types.TipSet) (struct) + case "current": + + { + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Current: %w", err) + } + + } + // t.Parent (types.TipSet) (struct) + case "parent": + + { + + if err := t.Parent.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Parent: %w", err) + } + + } + // t.BaseFee (big.Int) (struct) + case "basefee": + + { + + if err := t.BaseFee.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BaseFee: %w", err) + } + + } + // t.FilVested (big.Int) (struct) + case "filvested": + + { + + if err := t.FilVested.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilVested: %w", err) + } + + } + // t.FilMined (big.Int) (struct) + case "filmined": + + { + + if err := t.FilMined.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilMined: %w", err) + } + + } + // t.FilBurnt (big.Int) (struct) + case "filburnt": + + { + + if err := t.FilBurnt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilBurnt: %w", err) + } + + } + // t.FilLocked (big.Int) (struct) + case "fillocked": + + { + + if err := t.FilLocked.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilLocked: %w", err) + } + + } + // t.FilCirculating (big.Int) (struct) + case "filcirculating": + + { + + if err := t.FilCirculating.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilCirculating: %w", err) + } + + } + // t.FilReserveDisbursed (big.Int) (struct) + case "filreserveddisbursed": + + { + + if err := t.FilReserveDisbursed.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilReserveDisbursed: %w", err) + } + + } + // t.FullBlocks (cid.Cid) (struct) + case "fullblocks": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.FullBlocks: %w", err) + } + + t.FullBlocks = c + + } + // t.ImplicitMessages (cid.Cid) (struct) + case "implicitmessages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ImplicitMessages: %w", err) + } + + t.ImplicitMessages = c + + } + // t.Actors (cid.Cid) (struct) + case "actors": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Actors: %w", err) + } + + t.Actors = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/messages/cbor_gen.go b/pkg/transform/cbor/messages/cbor_gen.go new file mode 100644 index 000000000..c04814292 --- /dev/null +++ b/pkg/transform/cbor/messages/cbor_gen.go @@ -0,0 +1,667 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package messages + +import ( + "fmt" + "io" + "math" + "sort" + + chain "github.com/filecoin-project/lily/pkg/extract/chain" + types "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *FullBlockIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.BlockHeader (types.BlockHeader) (struct) + if len("block_header") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"block_header\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("block_header"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("block_header")); err != nil { + return err + } + + if err := t.BlockHeader.MarshalCBOR(cw); err != nil { + return err + } + + // t.SecpMessages (cid.Cid) (struct) + if len("secp_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"secp_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("secp_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("secp_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.SecpMessages); err != nil { + return xerrors.Errorf("failed to write cid field t.SecpMessages: %w", err) + } + + // t.BlsMessages (cid.Cid) (struct) + if len("bls_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"bls_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("bls_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("bls_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.BlsMessages); err != nil { + return xerrors.Errorf("failed to write cid field t.BlsMessages: %w", err) + } + + return nil +} + +func (t *FullBlockIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = FullBlockIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FullBlockIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.BlockHeader (types.BlockHeader) (struct) + case "block_header": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.BlockHeader = new(types.BlockHeader) + if err := t.BlockHeader.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BlockHeader pointer: %w", err) + } + } + + } + // t.SecpMessages (cid.Cid) (struct) + case "secp_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SecpMessages: %w", err) + } + + t.SecpMessages = c + + } + // t.BlsMessages (cid.Cid) (struct) + case "bls_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.BlsMessages: %w", err) + } + + t.BlsMessages = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Message (types.Message) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (chain.ChainMessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.VmMessagesAmt (cid.Cid) (struct) + if len("vm_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"vm_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vm_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("vm_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.VmMessagesAmt); err != nil { + return xerrors.Errorf("failed to write cid field t.VmMessagesAmt: %w", err) + } + + return nil +} + +func (t *ChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = ChainMessageIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ChainMessageIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Message (types.Message) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.Message) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (chain.ChainMessageReceipt) (struct) + case "receipt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Receipt = new(chain.ChainMessageReceipt) + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) + } + } + + } + // t.VmMessagesAmt (cid.Cid) (struct) + case "vm_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VmMessagesAmt: %w", err) + } + + t.VmMessagesAmt = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *SignedChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Message (types.SignedMessage) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (chain.ChainMessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.VmMessagesAmt (cid.Cid) (struct) + if len("vm_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"vm_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vm_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("vm_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.VmMessagesAmt); err != nil { + return xerrors.Errorf("failed to write cid field t.VmMessagesAmt: %w", err) + } + + return nil +} + +func (t *SignedChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = SignedChainMessageIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SignedChainMessageIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Message (types.SignedMessage) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.SignedMessage) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (chain.ChainMessageReceipt) (struct) + case "receipt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Receipt = new(chain.ChainMessageReceipt) + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) + } + } + + } + // t.VmMessagesAmt (cid.Cid) (struct) + case "vm_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VmMessagesAmt: %w", err) + } + + t.VmMessagesAmt = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ImplicitMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Message (types.Message) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (chain.ImplicitMessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.VmMessagesAmt (cid.Cid) (struct) + if len("vm_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"vm_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vm_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("vm_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.VmMessagesAmt); err != nil { + return xerrors.Errorf("failed to write cid field t.VmMessagesAmt: %w", err) + } + + return nil +} + +func (t *ImplicitMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = ImplicitMessageIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ImplicitMessageIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Message (types.Message) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.Message) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (chain.ImplicitMessageReceipt) (struct) + case "receipt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Receipt = new(chain.ImplicitMessageReceipt) + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) + } + } + + } + // t.VmMessagesAmt (cid.Cid) (struct) + case "vm_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VmMessagesAmt: %w", err) + } + + t.VmMessagesAmt = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/messages/serialize.go b/pkg/transform/cbor/messages/serialize.go new file mode 100644 index 000000000..fe6c762fb --- /dev/null +++ b/pkg/transform/cbor/messages/serialize.go @@ -0,0 +1,255 @@ +package messages + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + adtstore "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/chain" +) + +type FullBlockIPLDContainer struct { + BlockHeader *types.BlockHeader `cborgen:"block_header"` + SecpMessages cid.Cid `cborgen:"secp_messages"` + BlsMessages cid.Cid `cborgen:"bls_messages"` +} + +func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[cid.Cid]*chain.FullBlock) (cid.Cid, error) { + fullBlkHamt, err := adt.MakeEmptyMap(store, 5) + if err != nil { + return cid.Undef, err + } + + for blkCid, fb := range fullBlks { + blsMsgHamt, err := MakeChainMessagesHAMT(ctx, store, fb.BlsMessages) + if err != nil { + return cid.Undef, err + } + + secpMsgHamt, err := MakeSignedChainMessagesHAMT(ctx, store, fb.SecpMessages) + if err != nil { + return cid.Undef, err + } + + if err := fullBlkHamt.Put(abi.CidKey(blkCid), &FullBlockIPLDContainer{ + BlockHeader: fb.Block, + SecpMessages: secpMsgHamt, + BlsMessages: blsMsgHamt, + }); err != nil { + return cid.Undef, err + } + } + + return fullBlkHamt.Root() +} + +func DecodeFullBlockHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) (map[cid.Cid]*chain.FullBlock, error) { + fullBlkHamt, err := adt.AsMap(store, root, 5) + if err != nil { + return nil, err + } + out := make(map[cid.Cid]*chain.FullBlock) + fbc := new(FullBlockIPLDContainer) + if err := fullBlkHamt.ForEach(fbc, func(key string) error { + chainMessages, err := DecodeChainMessagesHAMT(ctx, store, fbc.BlsMessages) + if err != nil { + return err + } + signedChainMessages, err := DecodeSignedChainMessagesHAMT(ctx, store, fbc.SecpMessages) + if err != nil { + return err + } + bh := new(types.BlockHeader) + *bh = *fbc.BlockHeader + // TODO assert key == bh.Cid + out[bh.Cid()] = &chain.FullBlock{ + Block: bh, + SecpMessages: signedChainMessages, + BlsMessages: chainMessages, + } + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type ChainMessageIPLDContainer struct { + Message *types.Message `cborgen:"message"` + Receipt *chain.ChainMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` +} + +func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*chain.ChainMessage) (cid.Cid, error) { + messageHamt, err := adt.MakeEmptyMap(store, 5) + if err != nil { + return cid.Undef, err + } + + for _, msg := range messages { + vmMsgRoot, err := msg.VmMessages.ToAdtArray(store, 5) + if err != nil { + return cid.Undef, err + } + if err := messageHamt.Put(abi.CidKey(msg.Message.Cid()), &ChainMessageIPLDContainer{ + Message: msg.Message, + Receipt: msg.Receipt, + VmMessagesAmt: vmMsgRoot, + }); err != nil { + return cid.Undef, err + } + } + return messageHamt.Root() +} + +func DecodeChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*chain.ChainMessage, error) { + messagesHamt, err := adt.AsMap(store, root, 5) + if err != nil { + return nil, err + } + + var out []*chain.ChainMessage + mc := new(ChainMessageIPLDContainer) + if err := messagesHamt.ForEach(mc, func(key string) error { + var msg types.Message + var rec chain.ChainMessageReceipt + + msg = *mc.Message + if mc.Receipt != nil { + rec = *mc.Receipt + } + vmMessages, err := chain.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + if err != nil { + return err + } + out = append(out, &chain.ChainMessage{ + Message: &msg, + Receipt: &rec, + VmMessages: vmMessages, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type SignedChainMessageIPLDContainer struct { + Message *types.SignedMessage `cborgen:"message"` + Receipt *chain.ChainMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` +} + +func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*chain.SignedChainMessage) (cid.Cid, error) { + messageHamt, err := adt.MakeEmptyMap(store, 5) + if err != nil { + return cid.Undef, err + } + + for _, msg := range messages { + vmMsgRoot, err := msg.VmMessages.ToAdtArray(store, 5) + if err != nil { + return cid.Undef, err + } + if err := messageHamt.Put(abi.CidKey(msg.Message.Cid()), &SignedChainMessageIPLDContainer{ + Message: msg.Message, + Receipt: msg.Receipt, + VmMessagesAmt: vmMsgRoot, + }); err != nil { + return cid.Undef, err + } + } + return messageHamt.Root() +} + +func DecodeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*chain.SignedChainMessage, error) { + messagesHamt, err := adt.AsMap(store, root, 5) + if err != nil { + return nil, err + } + + var out []*chain.SignedChainMessage + mc := new(SignedChainMessageIPLDContainer) + if err := messagesHamt.ForEach(mc, func(key string) error { + var msg types.SignedMessage + var rec chain.ChainMessageReceipt + + msg = *mc.Message + if mc.Receipt != nil { + rec = *mc.Receipt + } + vmMessages, err := chain.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + if err != nil { + return err + } + out = append(out, &chain.SignedChainMessage{ + Message: &msg, + Receipt: &rec, + VmMessages: vmMessages, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type ImplicitMessageIPLDContainer struct { + Message *types.Message `cborgen:"message"` + Receipt *chain.ImplicitMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` +} + +// MakeImplicitMessagesHAMT returns the root of a hamt node containing the set of implicit messages +func MakeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*chain.ImplicitMessage) (cid.Cid, error) { + messageHamt, err := adt.MakeEmptyMap(store, 5) + if err != nil { + return cid.Undef, err + } + + for _, msg := range messages { + vmMsgRoot, err := msg.VmMessages.ToAdtArray(store, 5) + if err != nil { + return cid.Undef, err + } + if err := messageHamt.Put(abi.CidKey(msg.Message.Cid()), &ImplicitMessageIPLDContainer{ + Message: msg.Message, + Receipt: msg.Receipt, + VmMessagesAmt: vmMsgRoot, + }); err != nil { + return cid.Undef, err + } + } + return messageHamt.Root() +} + +func DecodeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*chain.ImplicitMessage, error) { + messagesHamt, err := adt.AsMap(store, root, 5) + if err != nil { + return nil, err + } + + var out []*chain.ImplicitMessage + msg := new(ImplicitMessageIPLDContainer) + if err := messagesHamt.ForEach(msg, func(key string) error { + m := *msg.Message + rect := *msg.Receipt + vmMessages, err := chain.VmMessageListFromAdtArray(store, msg.VmMessagesAmt, 5) + if err != nil { + return err + } + out = append(out, &chain.ImplicitMessage{ + Message: &m, + Receipt: &rect, + VmMessages: vmMessages, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go new file mode 100644 index 000000000..15818a080 --- /dev/null +++ b/pkg/transform/cbor/router.go @@ -0,0 +1,279 @@ +package cbor + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/filecoin-project/go-state-types/abi" + adtStore "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + v1car "github.com/ipld/go-car" + "github.com/ipld/go-car/util" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" + "gorm.io/gorm/schema" + + "gorm.io/driver/postgres" + "gorm.io/gorm" + + "github.com/filecoin-project/lily/pkg/extract" + cboractors "github.com/filecoin-project/lily/pkg/transform/cbor/actors" + cbormessages "github.com/filecoin-project/lily/pkg/transform/cbor/messages" +) + +var log = logging.Logger("lily/transform/cbor") + +type RootStateIPLD struct { + StateVersion uint64 `cborgen:"stateversion"` + + NetworkName string `cborgen:"networkname"` + NetworkVersion uint64 `cborgen:"networkversion"` + + State cid.Cid `cborgen:"state"` // StateExtractionIPLD +} + +type RootStateModel struct { + gorm.Model + Height uint64 + Cid string + StateVersion uint64 + NetworkName string + NetworkVersion uint64 + StateExtraction string +} + +func (r *RootStateIPLD) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.Int64("state_version", int64(r.StateVersion)), + attribute.String("state_root", r.State.String()), + attribute.String("network_name", r.NetworkName), + attribute.Int64("network_version", int64(r.NetworkVersion)), + } +} + +func (r *RootStateIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range r.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + +type StateExtractionIPLD struct { + Current types.TipSet `cborgen:"current"` + Parent types.TipSet `cborgen:"parent"` + + BaseFee abi.TokenAmount `cborgen:"basefee"` + FilVested abi.TokenAmount `cborgen:"filvested"` + FilMined abi.TokenAmount `cborgen:"filmined"` + FilBurnt abi.TokenAmount `cborgen:"filburnt"` + FilLocked abi.TokenAmount `cborgen:"fillocked"` + FilCirculating abi.TokenAmount `cborgen:"filcirculating"` + FilReserveDisbursed abi.TokenAmount `cborgen:"filreserveddisbursed"` + + FullBlocks cid.Cid `cborgen:"fullblocks"` + ImplicitMessages cid.Cid `cborgen:"implicitmessages"` + Actors cid.Cid `cborgen:"actors"` +} + +type StateExtractionModel struct { + gorm.Model + Height uint64 + CurrentTipSet string + ParentTipSet string + BaseFee string + FullBlocks string + ImplicitMessages string + Actors string +} + +func (s *StateExtractionIPLD) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("current_tipset", s.Current.Key().String()), + attribute.String("parent_tipset", s.Parent.Key().String()), + attribute.String("base_fee", s.BaseFee.String()), + attribute.String("full_block_root", s.FullBlocks.String()), + attribute.String("implicit_message_root", s.ImplicitMessages.String()), + attribute.String("actors_root", s.Actors.String()), + } +} + +func (s *StateExtractionIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range s.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + +func NewTransformer(dir, prefix string) (*Transformer, error) { + if dirInfo, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("path (%s) does not exist: %w", dir, err) + } + if !dirInfo.IsDir() { + return nil, fmt.Errorf("path (%s) is not a directory", dir) + } + } + return &Transformer{carDirectory: dir, prefix: prefix}, nil +} + +type Transformer struct { + carDirectory string + prefix string +} + +func (t *Transformer) Persist(ctx context.Context, chainState *extract.ChainState) error { + start := time.Now() + carPath := filepath.Join(t.carDirectory, fmt.Sprintf("%s_%d_%d.car", t.prefix, chainState.Parent.Height(), chainState.Current.Height())) + f, err := os.OpenFile(carPath, os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + bs := blockstore.NewMemorySync() + root, err := PersistToStore(ctx, bs, chainState) + if err != nil { + return err + } + bw := bufio.NewWriterSize(f, 1<<20) + if err := WriteCarV1(ctx, root, bs, bw); err != nil { + return err + } + log.Infow("created chain delta", "path", carPath, "duration", time.Since(start)) + return nil +} + +func WriteCarV1(ctx context.Context, root cid.Cid, bs blockstore.Blockstore, w io.Writer) error { + if err := v1car.WriteHeader(&v1car.CarHeader{ + Roots: []cid.Cid{root}, + Version: 1, + }, w); err != nil { + return err + } + + keyCh, err := bs.AllKeysChan(ctx) + if err != nil { + return err + } + + for key := range keyCh { + blk, err := bs.Get(ctx, key) + if err != nil { + return err + } + if err := util.LdWrite(w, blk.Cid().Bytes(), blk.RawData()); err != nil { + return err + } + } + return nil +} + +func PersistToStore(ctx context.Context, bs blockstore.Blockstore, chainState *extract.ChainState) (cid.Cid, error) { + store := adtStore.WrapBlockStore(ctx, bs) + + implicitMsgsAMT, err := cbormessages.MakeImplicitMessagesHAMT(ctx, store, chainState.Message.ImplicitMessages) + if err != nil { + return cid.Undef, err + } + + fullBlkHAMT, err := cbormessages.MakeFullBlockHAMT(ctx, store, chainState.Message.FullBlocks) + if err != nil { + return cid.Undef, err + } + + actorStateContainer, err := cboractors.ProcessActorsStates(ctx, store, chainState.Actors) + if err != nil { + return cid.Undef, err + } + + actorStatesRoot, err := store.Put(ctx, actorStateContainer) + if err != nil { + return cid.Undef, err + } + + extractedState := &StateExtractionIPLD{ + Current: *chainState.Current, + Parent: *chainState.Parent, + BaseFee: chainState.Message.BaseFee, + FilVested: chainState.Message.CirculatingSupply.FilVested, + FilMined: chainState.Message.CirculatingSupply.FilMined, + FilBurnt: chainState.Message.CirculatingSupply.FilBurnt, + FilLocked: chainState.Message.CirculatingSupply.FilLocked, + FilCirculating: chainState.Message.CirculatingSupply.FilCirculating, + FilReserveDisbursed: chainState.Message.CirculatingSupply.FilReserveDisbursed, + FullBlocks: fullBlkHAMT, + ImplicitMessages: implicitMsgsAMT, + Actors: actorStatesRoot, + } + + extractedStateRoot, err := store.Put(ctx, extractedState) + if err != nil { + return cid.Undef, err + } + + rootState := &RootStateIPLD{ + NetworkVersion: chainState.NetworkVersion, + NetworkName: chainState.NetworkName, + StateVersion: 0, + State: extractedStateRoot, + } + + root, err := store.Put(ctx, rootState) + if err != nil { + return cid.Undef, err + } + dsn := "host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable" + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "lily_cbor_", + SingularTable: false, + }, + }) + if err != nil { + return cid.Undef, err + } + if err := db.AutoMigrate(&RootStateModel{}, &StateExtractionModel{}, &cboractors.ActorStateModel{}); err != nil { + return cid.Undef, err + } + rootModel := RootStateModel{ + Height: uint64(chainState.Current.Height()), + Cid: root.String(), + StateVersion: rootState.StateVersion, + NetworkName: rootState.NetworkName, + NetworkVersion: rootState.NetworkVersion, + StateExtraction: rootState.State.String(), + } + stateModel := StateExtractionModel{ + Height: uint64(chainState.Current.Height()), + CurrentTipSet: chainState.Current.String(), + ParentTipSet: chainState.Parent.String(), + BaseFee: chainState.Message.BaseFee.String(), + FullBlocks: fullBlkHAMT.String(), + ImplicitMessages: implicitMsgsAMT.String(), + Actors: actorStatesRoot.String(), + } + if err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Create(&rootModel).Error; err != nil { + return err + } + if err := tx.Create(&stateModel).Error; err != nil { + return err + } + as := actorStateContainer.AsModel() + as.Height = uint64(chainState.Current.Height()) + if err := tx.Create(as).Error; err != nil { + return err + } + return nil + }); err != nil { + return cid.Undef, err + } + return root, nil +} diff --git a/pkg/transform/gorm/lambda/params.go b/pkg/transform/gorm/lambda/params.go new file mode 100644 index 000000000..7e3914b32 --- /dev/null +++ b/pkg/transform/gorm/lambda/params.go @@ -0,0 +1,42 @@ +package lambda + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + "gorm.io/gorm" + + "github.com/filecoin-project/lily/lens/util" + "github.com/filecoin-project/lily/pkg/transform/gorm/models" + "github.com/filecoin-project/lily/tasks" +) + +func ParseParams(ctx context.Context, api tasks.DataSource, db *gorm.DB) error { + var messages []models.Message + res := db.Find(&messages) + if res.Error != nil { + return res.Error + } + out := make([]models.ParsedMessageParams, 0, len(messages)) + for _, msg := range messages { + act, err := api.Actor(ctx, msg.To.Addr, types.EmptyTSK) + if err != nil { + return err + } + params, method, err := util.ParseParams(msg.Params, abi.MethodNum(msg.Method), act.Code) + if err != nil { + // TODO could continue + return err + } + out = append(out, models.ParsedMessageParams{ + Cid: msg.Cid, + Params: params, + Method: method, + }) + } + if err := db.Create(out).Error; err != nil { + return err + } + return nil +} diff --git a/pkg/transform/gorm/models/blockheader.go b/pkg/transform/gorm/models/blockheader.go new file mode 100644 index 000000000..e13411108 --- /dev/null +++ b/pkg/transform/gorm/models/blockheader.go @@ -0,0 +1,18 @@ +package models + +import ( + types "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +type BlockHeaderModel struct { + Cid types.DbCID `gorm:"primaryKey"` + StateRoot types.DbCID + Height int64 + Miner types.DbAddr + ParentWeight types.DbBigInt + TimeStamp uint64 + ForkSignaling uint64 + BaseFee types.DbToken + WinCount int64 + ParentBaseFee types.DbToken +} diff --git a/pkg/transform/gorm/models/chainmessage.go b/pkg/transform/gorm/models/chainmessage.go new file mode 100644 index 000000000..bc39954dc --- /dev/null +++ b/pkg/transform/gorm/models/chainmessage.go @@ -0,0 +1,25 @@ +package models + +import ( + types "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +type Message struct { + Cid types.DbCID `gorm:"primaryKey"` + Version int64 + To types.DbAddr + From types.DbAddr + Nonce uint64 + Value types.DbToken + GasLimit int64 + GasFeeCap types.DbToken + GasPremium types.DbToken + Method uint64 + Params []byte +} + +type ParsedMessageParams struct { + Cid types.DbCID `gorm:"primaryKey"` + Params string `gorm:"jsonb"` + Method string +} diff --git a/pkg/transform/gorm/models/receipt.go b/pkg/transform/gorm/models/receipt.go new file mode 100644 index 000000000..827fa22dd --- /dev/null +++ b/pkg/transform/gorm/models/receipt.go @@ -0,0 +1,26 @@ +package models + +import ( + types "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +type MessageReceipt struct { + MessageCid types.DbCID `gorm:"primaryKey"` + Receipt Receipt `gorm:"embedded"` + + BaseFeeBurn types.DbToken + OverEstimationBurn types.DbToken + MinerPenalty types.DbToken + MinerTip types.DbToken + Refund types.DbToken + GasRefund int64 + GasBurned int64 + Error string +} + +type Receipt struct { + Index int64 `gorm:"primaryKey"` + ExitCode int64 + GasUsed int64 + Return []byte +} diff --git a/pkg/transform/gorm/models/vmmessages.go b/pkg/transform/gorm/models/vmmessages.go new file mode 100644 index 000000000..2b44f7edc --- /dev/null +++ b/pkg/transform/gorm/models/vmmessages.go @@ -0,0 +1,15 @@ +package models + +import "github.com/filecoin-project/lily/pkg/transform/gorm/types" + +type VmMessage struct { + Source types.DbCID `gorm:"primaryKey"` + Cid types.DbCID `gorm:"primaryKey"` + To types.DbAddr + From types.DbAddr + Value types.DbToken + Method uint64 + Params []byte + Receipt Receipt `gorm:"embedded"` + Error string +} diff --git a/pkg/transform/gorm/run.go b/pkg/transform/gorm/run.go new file mode 100644 index 000000000..516512f37 --- /dev/null +++ b/pkg/transform/gorm/run.go @@ -0,0 +1,251 @@ +package gorm + +import ( + "context" + "fmt" + "io" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + v1car "github.com/ipld/go-car" + "gorm.io/gorm" + + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" + "github.com/filecoin-project/lily/pkg/transform/gorm/models" + dbtypes "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +func Run(ctx context.Context, r io.Reader, db *gorm.DB) error { + // create a blockstore and load the contents of the car file (reader is a pointer to the car file) into it. + bs := blockstore.NewMemorySync() + header, err := v1car.LoadCar(ctx, bs, r) + if err != nil { + return err + } + // expect to have a single root (cid) pointing to content + if len(header.Roots) != 1 { + return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) + } + + // we need to wrap the blockstore to meet some dumb interface. + s := store.WrapBlockStore(ctx, bs) + + // load the root container, this contains netwwork metadata and the root (cid) of the extracted state. + rootIPLDContainer := new(cbor.RootStateIPLD) + if err := s.Get(ctx, header.Roots[0], rootIPLDContainer); err != nil { + return err + } + + // load the extracted state, this contains links to fullblocks, implicit message, network economics, and actor states + stateExtractionIPLDContainer := new(cbor.StateExtractionIPLD) + if err := s.Get(ctx, rootIPLDContainer.State, stateExtractionIPLDContainer); err != nil { + return err + } + + // ohh and it also contains the tipset the extraction was performed over. + current := &stateExtractionIPLDContainer.Current + parent := &stateExtractionIPLDContainer.Parent + + // for now we will only handle the fullblock dag. + if err := HandleFullBlocks(ctx, db, s, current, parent, stateExtractionIPLDContainer.FullBlocks); err != nil { + return err + } + + return nil +} + +func HandleFullBlocks(ctx context.Context, db *gorm.DB, s store.Store, current, parent *types.TipSet, root cid.Cid) error { + // decode the content at the root (cid) into a concrete type we can inspect, a map of block CID to the block, its messages, their receipts and vm messages. + fullBlocks, err := messages.DecodeFullBlockHAMT(ctx, s, root) + if err != nil { + return err + } + // migrate the database, this creates new tables or updates existing ones with new fields + if err := db.AutoMigrate(&models.BlockHeaderModel{}, &models.Message{}, models.MessageReceipt{}, models.VmMessage{}); err != nil { + return err + } + // make some blockheaders and plop em in the database + if err := db.Create(MakeBlockHeaderModels(ctx, fullBlocks)).Error; err != nil { + return err + } + // now do messages + if err := db.Create(MakeMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + // finally receipts + if err := db.Create(MakeReceipts(ctx, fullBlocks)).Error; err != nil { + return err + } + + if err := db.Create(MakeVmMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + + // okay all done. + return nil +} + +func MakeBlockHeaderModels(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.BlockHeaderModel { + out := make([]*models.BlockHeaderModel, 0, len(fullBlocks)) + for _, fb := range fullBlocks { + out = append(out, &models.BlockHeaderModel{ + Cid: dbtypes.DbCID{CID: fb.Block.Cid()}, + StateRoot: dbtypes.DbCID{CID: fb.Block.ParentStateRoot}, + Height: int64(fb.Block.Height), + Miner: dbtypes.DbAddr{Addr: fb.Block.Miner}, + ParentWeight: dbtypes.DbBigInt{BigInt: fb.Block.ParentWeight}, + TimeStamp: fb.Block.Timestamp, + ForkSignaling: fb.Block.ForkSignaling, + BaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, + WinCount: fb.Block.ElectionProof.WinCount, + ParentBaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, + }) + } + return out +} + +func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.Message { + // messages can be contained in more than 1 block, this is used to prevent persisting them twice + seen := cid.NewSet() + var out []*models.Message + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + if !seen.Visit(smsg.Message.Cid()) { + continue + } + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: smsg.Message.Cid()}, + Version: int64(smsg.Message.Message.Version), + To: dbtypes.DbAddr{Addr: smsg.Message.Message.To}, + From: dbtypes.DbAddr{smsg.Message.Message.From}, + Nonce: smsg.Message.Message.Nonce, + Value: dbtypes.DbToken{Token: smsg.Message.Message.Value}, + GasLimit: smsg.Message.Message.GasLimit, + GasFeeCap: dbtypes.DbToken{Token: smsg.Message.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: smsg.Message.Message.GasPremium}, + Method: uint64(smsg.Message.Message.Method), + Params: smsg.Message.Message.Params, + }) + } + for _, msg := range fb.BlsMessages { + if !seen.Visit(msg.Message.Cid()) { + continue + } + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: msg.Message.Cid()}, + Version: int64(msg.Message.Version), + To: dbtypes.DbAddr{Addr: msg.Message.To}, + From: dbtypes.DbAddr{msg.Message.From}, + Nonce: msg.Message.Nonce, + Value: dbtypes.DbToken{Token: msg.Message.Value}, + GasLimit: msg.Message.GasLimit, + GasFeeCap: dbtypes.DbToken{Token: msg.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: msg.Message.GasPremium}, + Method: uint64(msg.Message.Method), + Params: msg.Message.Params, + }) + } + } + return out +} + +func MakeReceipts(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.MessageReceipt { + var out []*models.MessageReceipt + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + // TODO this is buggy + if smsg.Receipt.GasOutputs == nil { + continue + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: smsg.Message.Cid()}, + Receipt: models.Receipt{ + Index: smsg.Receipt.Index, + ExitCode: int64(smsg.Receipt.Receipt.ExitCode), + GasUsed: smsg.Receipt.Receipt.GasUsed, + Return: smsg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.Refund}, + GasRefund: smsg.Receipt.GasOutputs.GasRefund, + GasBurned: smsg.Receipt.GasOutputs.GasBurned, + }) + } + for _, msg := range fb.BlsMessages { + if msg.Receipt.GasOutputs == nil { + continue + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: msg.Message.Cid()}, + Receipt: models.Receipt{ + Index: msg.Receipt.Index, + ExitCode: int64(msg.Receipt.Receipt.ExitCode), + GasUsed: msg.Receipt.Receipt.GasUsed, + Return: msg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.Refund}, + GasRefund: msg.Receipt.GasOutputs.GasRefund, + GasBurned: msg.Receipt.GasOutputs.GasBurned, + }) + } + } + return out +} + +func MakeVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.VmMessage { + var out []*models.VmMessage + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + for _, msg := range fb.BlsMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + } + return out +} diff --git a/pkg/transform/gorm/types/address.go b/pkg/transform/gorm/types/address.go new file mode 100644 index 000000000..858933bb2 --- /dev/null +++ b/pkg/transform/gorm/types/address.go @@ -0,0 +1,33 @@ +package types + +import ( + "database/sql/driver" + "fmt" + + "github.com/filecoin-project/go-address" +) + +//copied from https://github.com/application-research/estuary/blob/f740b018af1d7ef6eaafb51d1c7b18a2bda0b589/util/database.go#L63 + +type DbAddr struct { + Addr address.Address +} + +func (dba *DbAddr) Scan(v interface{}) error { + s, ok := v.(string) + if !ok { + return fmt.Errorf("DbAddrs must be strings") + } + + addr, err := address.NewFromString(s) + if err != nil { + return err + } + + dba.Addr = addr + return nil +} + +func (dba DbAddr) Value() (driver.Value, error) { + return dba.Addr.String(), nil +} diff --git a/pkg/transform/gorm/types/bigint.go b/pkg/transform/gorm/types/bigint.go new file mode 100644 index 000000000..bd7915a23 --- /dev/null +++ b/pkg/transform/gorm/types/bigint.go @@ -0,0 +1,35 @@ +package types + +import ( + "database/sql/driver" + "fmt" + + "github.com/filecoin-project/go-state-types/big" +) + +type DbBigInt struct { + BigInt big.Int +} + +func (dbb *DbBigInt) Scan(v interface{}) error { + s, ok := v.(string) + if !ok { + return fmt.Errorf("DbBigInts must be strings") + } + + bi, err := big.FromString(s) + if err != nil { + return err + } + + dbb.BigInt = bi + return nil +} + +func (dbb DbBigInt) Value() (driver.Value, error) { + return dbb.BigInt.String(), nil +} + +func (DbBigInt) GormDataType() string { + return "bigint" +} diff --git a/pkg/transform/gorm/types/cid.go b/pkg/transform/gorm/types/cid.go new file mode 100644 index 000000000..649945174 --- /dev/null +++ b/pkg/transform/gorm/types/cid.go @@ -0,0 +1,57 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "fmt" + + "github.com/ipfs/go-cid" +) + +// copied from https://github.com/application-research/estuary/blob/f740b018af1d7ef6eaafb51d1c7b18a2bda0b589/util/database.go#L63 + +type DbCID struct { + CID cid.Cid +} + +func (dbc *DbCID) Scan(v interface{}) error { + b, ok := v.(string) + if !ok { + return fmt.Errorf("dbcids must be strings") + } + + if len(b) == 0 { + return nil + } + + c, err := cid.Decode(b) + if err != nil { + return err + } + + dbc.CID = c + return nil +} + +func (dbc DbCID) Value() (driver.Value, error) { + return dbc.CID.String(), nil +} + +func (dbc DbCID) MarshalJSON() ([]byte, error) { + return json.Marshal(dbc.CID.String()) +} + +func (dbc *DbCID) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + c, err := cid.Decode(s) + if err != nil { + return err + } + + dbc.CID = c + return nil +} diff --git a/pkg/transform/gorm/types/json.go b/pkg/transform/gorm/types/json.go new file mode 100644 index 000000000..1853acfa4 --- /dev/null +++ b/pkg/transform/gorm/types/json.go @@ -0,0 +1,28 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "errors" +) + +// JSONB Interface for JSONB Field of yourTableName Table +type DbJson string + +// Value Marshal +func (a DbJson) Value() (driver.Value, error) { + return json.Marshal(a) +} + +// Scan Unmarshal +func (a *DbJson) Scan(value interface{}) error { + b, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + return json.Unmarshal(b, &a) +} + +func (DbJson) GormDataType() string { + return "jsonb" +} diff --git a/pkg/transform/gorm/types/token.go b/pkg/transform/gorm/types/token.go new file mode 100644 index 000000000..549fb2800 --- /dev/null +++ b/pkg/transform/gorm/types/token.go @@ -0,0 +1,48 @@ +package types + +import ( + "database/sql/driver" + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" +) + +type DbToken struct { + Token abi.TokenAmount +} + +func (dbt *DbToken) Scan(v interface{}) error { + switch v.(type) { + case string: + s, ok := v.(string) + if !ok { + return fmt.Errorf("DbToken must be strings") + } + + token, err := big.FromString(s) + if err != nil { + return err + } + dbt.Token = token + return nil + case int64: + s, ok := v.(int64) + if !ok { + return fmt.Errorf("DbToken must be strings") + } + + token := big.NewInt(s) + dbt.Token = token + return nil + } + panic("here") +} + +func (dbt DbToken) Value() (driver.Value, error) { + return dbt.Token.String(), nil +} + +func (DbToken) GormDataType() string { + return "bigint" +} diff --git a/pkg/transform/timescale/actors/init/addresses.go b/pkg/transform/timescale/actors/init/addresses.go new file mode 100644 index 000000000..533993b1c --- /dev/null +++ b/pkg/transform/timescale/actors/init/addresses.go @@ -0,0 +1,49 @@ +package init + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + initmodel "github.com/filecoin-project/lily/model/actors/init" + "github.com/filecoin-project/lily/pkg/core" + initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Addresses struct{} + +func (a Addresses) Extract(ctx context.Context, current, executed *types.TipSet, changes *initdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.IDAddress, current) + for _, change := range changes.AddressesChanges { + if change.Change == core.ChangeTypeAdd || change.Change == core.ChangeTypeModify { + robustAddr, err := address.NewFromBytes(change.Address) + if err != nil { + report.AddError(err) + continue + } + var actorID cbg.CborInt + if err := actorID.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + idAddr, err := address.NewIDAddress(uint64(actorID)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&initmodel.IDAddress{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + ID: idAddr.String(), + Address: robustAddr.String(), + }) + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/init/processor.go b/pkg/transform/timescale/actors/init/processor.go new file mode 100644 index 000000000..36326c6cd --- /dev/null +++ b/pkg/transform/timescale/actors/init/processor.go @@ -0,0 +1,35 @@ +package init + +import ( + "context" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" + initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +func TransformInitState(ctx context.Context, s store.Store, current, executed *types.TipSet, initMapRoot *cid.Cid) (model.Persistable, error) { + if initMapRoot == nil { + return data.StartProcessingReport(tasktype.IDAddress, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), nil + } + initState := new(initdiff.StateChange) + if err := s.Get(ctx, *initMapRoot, initState); err != nil { + return nil, err + } + + initStateDiff, err := initState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + + return Addresses{}.Extract(ctx, current, executed, initStateDiff), nil +} diff --git a/pkg/transform/timescale/actors/market/router.go b/pkg/transform/timescale/actors/market/router.go new file mode 100644 index 000000000..4a2979cd2 --- /dev/null +++ b/pkg/transform/timescale/actors/market/router.go @@ -0,0 +1,46 @@ +package market + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +func TransformMarketState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + return model.PersistableList{ + data.StartProcessingReport(tasktype.MarketDealState, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MarketDealProposal, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8, + actortypes.Version9: + return v1.TransformMarketState(ctx, s, version, current, executed, *root) + } + return nil, fmt.Errorf("unsupported version : %d", version) + +} diff --git a/pkg/transform/timescale/actors/market/util/label.go b/pkg/transform/timescale/actors/market/util/label.go new file mode 100644 index 000000000..70c932fcf --- /dev/null +++ b/pkg/transform/timescale/actors/market/util/label.go @@ -0,0 +1,24 @@ +package util + +import ( + "strings" + "unicode/utf8" + + "golang.org/x/text/runes" +) + +// SanitizeLabel ensures: +// - s is a valid utf8 string by removing any ill formed bytes. +// - s does not contain any nil (\x00) bytes because postgres doesn't support storing NULL (\0x00) characters in text fields. +func SanitizeLabel(s string) string { + if s == "" { + return s + } + s = strings.Replace(s, "\000", "", -1) + if utf8.ValidString(s) { + return s + } + + tr := runes.ReplaceIllFormed() + return tr.String(s) +} diff --git a/pkg/transform/timescale/actors/market/v1/router.go b/pkg/transform/timescale/actors/market/v1/router.go new file mode 100644 index 000000000..66d340d82 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/router.go @@ -0,0 +1,99 @@ +package v1 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v9" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable +} + +func TransformMarketState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + marketState := new(marketdiff.StateChange) + if err := s.Get(ctx, root, marketState); err != nil { + return nil, err + } + marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + transformers, err := LookupMarketStateTransformer(version) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m := t.Transform(ctx, current, executed, marketStateDiff) + out = append(out, m) + } + return out, nil +} + +func LookupMarketStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Deals{}, + v1_0.Proposals{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Deals{}, + v1_2.Proposals{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Deals{}, + v1_3.Proposals{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Deals{}, + v1_4.Proposals{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Deals{}, + v1_5.Proposals{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Deals{}, + v1_6.Proposals{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Deals{}, + v1_7.Proposals{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Deals{}, + v1_8.Proposals{}, + }, nil + case actortypes.Version9: + return []Transformer{ + v1_9.Deals{}, + v1_9.Proposals{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for market transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/market/v1/v0/deals.go b/pkg/transform/timescale/actors/market/v1/v0/deals.go new file mode 100644 index 000000000..53a279736 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v0/deals.go @@ -0,0 +1,59 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v0/proposals.go b/pkg/transform/timescale/actors/market/v1/v0/proposals.go new file mode 100644 index 000000000..75ddd4a91 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v0/proposals.go @@ -0,0 +1,72 @@ +package v0 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v2/deals.go b/pkg/transform/timescale/actors/market/v1/v2/deals.go new file mode 100644 index 000000000..ac3a661c1 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v2/deals.go @@ -0,0 +1,59 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v2/proposals.go b/pkg/transform/timescale/actors/market/v1/v2/proposals.go new file mode 100644 index 000000000..b771b392d --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v2/proposals.go @@ -0,0 +1,72 @@ +package v2 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v3/deals.go b/pkg/transform/timescale/actors/market/v1/v3/deals.go new file mode 100644 index 000000000..56ac5f168 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v3/deals.go @@ -0,0 +1,59 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v3/proposals.go b/pkg/transform/timescale/actors/market/v1/v3/proposals.go new file mode 100644 index 000000000..af5be3976 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v3/proposals.go @@ -0,0 +1,72 @@ +package v3 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v4/deals.go b/pkg/transform/timescale/actors/market/v1/v4/deals.go new file mode 100644 index 000000000..6dd82d7d5 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v4/deals.go @@ -0,0 +1,59 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v4/proposals.go b/pkg/transform/timescale/actors/market/v1/v4/proposals.go new file mode 100644 index 000000000..f27c635a2 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v4/proposals.go @@ -0,0 +1,72 @@ +package v4 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v5/deals.go b/pkg/transform/timescale/actors/market/v1/v5/deals.go new file mode 100644 index 000000000..97b745ce0 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v5/deals.go @@ -0,0 +1,59 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v5/proposals.go b/pkg/transform/timescale/actors/market/v1/v5/proposals.go new file mode 100644 index 000000000..a02b55905 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v5/proposals.go @@ -0,0 +1,72 @@ +package v5 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v6/deals.go b/pkg/transform/timescale/actors/market/v1/v6/deals.go new file mode 100644 index 000000000..b742734e7 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v6/deals.go @@ -0,0 +1,59 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v6/proposals.go b/pkg/transform/timescale/actors/market/v1/v6/proposals.go new file mode 100644 index 000000000..f30bc6232 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v6/proposals.go @@ -0,0 +1,72 @@ +package v6 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v7/deals.go b/pkg/transform/timescale/actors/market/v1/v7/deals.go new file mode 100644 index 000000000..e9a4ad05b --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v7/deals.go @@ -0,0 +1,59 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v7/proposals.go b/pkg/transform/timescale/actors/market/v1/v7/proposals.go new file mode 100644 index 000000000..c2e7fa115 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v7/proposals.go @@ -0,0 +1,72 @@ +package v7 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v8/deals.go b/pkg/transform/timescale/actors/market/v1/v8/deals.go new file mode 100644 index 000000000..d1a5054e5 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v8/deals.go @@ -0,0 +1,59 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/builtin/v8/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v8/proposals.go b/pkg/transform/timescale/actors/market/v1/v8/proposals.go new file mode 100644 index 000000000..a7ce8d949 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v8/proposals.go @@ -0,0 +1,100 @@ +package v8 + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v8/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + m, err := MarketDealProposalChangesAsModel(ctx, current, marketProposals) + if err != nil { + return report.AddError(err).Finish() + } + return report.AddModels(m).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + var isString bool + var base64Label string + if prop.State.Label.IsString() { + labelString, err := prop.State.Label.ToString() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not a string despite claiming it is (developer error?)", prop.DealID) + } + + isString = true + base64Label = base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(labelString))) + + } else if prop.State.Label.IsBytes() { + labelBytes, err := prop.State.Label.ToBytes() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not bytes despit claiming it is (developer error?)", prop.DealID) + } + + isString = false + base64Label = base64.StdEncoding.EncodeToString(labelBytes) + + } else { + // TODO this should never happen, but if it does it indicates a logic. + return nil, fmt.Errorf("deal proposal (ID: %d) label is neither bytes nor string (DEVELOPER ERROR)", prop.DealID) + } + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: base64Label, + IsString: isString, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v1/v9/deals.go b/pkg/transform/timescale/actors/market/v1/v9/deals.go new file mode 100644 index 000000000..95a4358f4 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v9/deals.go @@ -0,0 +1,59 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel +} diff --git a/pkg/transform/timescale/actors/market/v1/v9/proposals.go b/pkg/transform/timescale/actors/market/v1/v9/proposals.go new file mode 100644 index 000000000..e02fd7957 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/v9/proposals.go @@ -0,0 +1,100 @@ +package v9 + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + m, err := MarketDealProposalChangesAsModel(ctx, current, marketProposals) + if err != nil { + return report.AddError(err).Finish() + } + return report.AddModels(m).Finish() +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + var isString bool + var base64Label string + if prop.State.Label.IsString() { + labelString, err := prop.State.Label.ToString() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not a string despite claiming it is (developer error?)", prop.DealID) + } + + isString = true + base64Label = base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(labelString))) + + } else if prop.State.Label.IsBytes() { + labelBytes, err := prop.State.Label.ToBytes() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not bytes despit claiming it is (developer error?)", prop.DealID) + } + + isString = false + base64Label = base64.StdEncoding.EncodeToString(labelBytes) + + } else { + // TODO this should never happen, but if it does it indicates a logic. + return nil, fmt.Errorf("deal proposal (ID: %d) label is neither bytes nor string (DEVELOPER ERROR)", prop.DealID) + } + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: base64Label, + IsString: isString, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go new file mode 100644 index 000000000..503550f18 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/router.go @@ -0,0 +1,62 @@ +package miner + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +func TransformMinerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + sectorInfoTaskName := tasktype.MinerSectorInfoV7 + if version < actortypes.Version7 { + sectorInfoTaskName = tasktype.MinerSectorInfoV1_6 + } + return model.PersistableList{ + data.StartProcessingReport(tasktype.MinerInfo, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MinerPreCommitInfo, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MinerSectorDeal, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MinerSectorEvent, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(sectorInfoTaskName, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8, + actortypes.Version9: + return v1.TransformMinerStates(ctx, s, version, current, executed, *root) + } + return nil, fmt.Errorf("unsupported version : %d", version) + +} diff --git a/pkg/transform/timescale/actors/miner/util/deals.go b/pkg/transform/timescale/actors/miner/util/deals.go new file mode 100644 index 000000000..48ee55b73 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/util/deals.go @@ -0,0 +1,31 @@ +package util + +import "github.com/filecoin-project/go-state-types/abi" + +func CompareDealIDs(cur, pre []abi.DealID) []abi.DealID { + var diff []abi.DealID + + // Loop two times, first to find cur dealIDs not in pre, + // second loop to find pre dealIDs not in cur + for i := 0; i < 2; i++ { + for _, s1 := range cur { + found := false + for _, s2 := range pre { + if s1 == s2 { + found = true + break + } + } + // DealID not found. We add it to return slice + if !found { + diff = append(diff, s1) + } + } + // Swap the slices, only if it was the first loop + if i == 0 { + cur, pre = pre, cur + } + } + + return diff +} diff --git a/pkg/transform/timescale/actors/miner/util/info.go b/pkg/transform/timescale/actors/miner/util/info.go new file mode 100644 index 000000000..89f8c6dd4 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/util/info.go @@ -0,0 +1,80 @@ +package util + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + "github.com/libp2p/go-libp2p/core/peer" + maddr "github.com/multiformats/go-multiaddr" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" +) + +type WorkerKeyChanges interface { + NewWorker() address.Address + EffectiveAt() abi.ChainEpoch +} + +type MinerInfo interface { + PendingWorkerKey() (WorkerKeyChanges, bool) + ControlAddresses() []address.Address + Multiaddrs() []abi.Multiaddrs + Owner() address.Address + Worker() address.Address + SectorSize() abi.SectorSize + PeerId() abi.PeerID +} + +func ExtractMinerInfo(ctx context.Context, current, executed *types.TipSet, addr address.Address, info MinerInfo) (model.Persistable, error) { + var newWorker string + var newWorkerEpoch int64 + if pendingWorkerKey, changed := info.PendingWorkerKey(); changed { + if pendingWorkerKey.NewWorker() != address.Undef { + newWorker = pendingWorkerKey.NewWorker().String() + } + newWorkerEpoch = int64(pendingWorkerKey.EffectiveAt()) + } + + var newCtrlAddresses []string + for _, addr := range info.ControlAddresses() { + newCtrlAddresses = append(newCtrlAddresses, addr.String()) + } + + // best effort to decode, we have no control over what miners put in this field, its just bytes. + var newMultiAddrs []string + for _, addr := range info.Multiaddrs() { + newMaddr, err := maddr.NewMultiaddrBytes(addr) + if err == nil { + newMultiAddrs = append(newMultiAddrs, newMaddr.String()) + } else { + //log.Debugw("failed to decode miner multiaddr", "miner", a.Address, "multiaddress", addr, "error", err) + } + } + mi := &minermodel.MinerInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + OwnerID: info.Owner().String(), + WorkerID: info.Worker().String(), + NewWorker: newWorker, + WorkerChangeEpoch: newWorkerEpoch, + ConsensusFaultedElapsed: -1, + ControlAddresses: newCtrlAddresses, + MultiAddresses: newMultiAddrs, + SectorSize: uint64(info.SectorSize()), + } + + if info.PeerId() != nil { + newPeerID, err := peer.IDFromBytes(info.PeerId()) + if err != nil { + //log.Warnw("failed to decode miner peerID", "miner", a.Address, "head", a.Actor.Head.String(), "error", err) + } else { + mi.PeerID = newPeerID.String() + } + } + + return mi, nil +} diff --git a/pkg/transform/timescale/actors/miner/v1/router.go b/pkg/transform/timescale/actors/miner/v1/router.go new file mode 100644 index 000000000..773ceb00d --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/router.go @@ -0,0 +1,154 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v9" +) + +func TransformMinerStates(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(s, root, 5) + if err != nil { + return nil, err + } + + // load map entires into a list to process below + var miners []*minertypes.MinerStateChange + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) + if err != nil { + return err + } + miners = append(miners, &minertypes.MinerStateChange{ + Address: addr, + StateChange: minerStateDiff, + }) + return nil + }); err != nil { + return nil, err + } + + // get a list of transformers capable of handling this actor version + transformers, err := LookupMinerStateTransformer(version) + if err != nil { + return nil, err + } + + // run each transformer keeping its model + out := model.PersistableList{} + for _, t := range transformers { + m := t.Transform(ctx, current, executed, miners) + out = append(out, m) + } + return out, nil +} + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable +} + +func LookupMinerStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Info{}, + v1_0.PreCommit{}, + v1_0.SectorDeal{}, + v1_0.SectorEvent{}, + v1_0.Sector{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Info{}, + v1_2.PreCommit{}, + v1_2.SectorDeal{}, + v1_2.SectorEvent{}, + v1_2.Sector{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Info{}, + v1_3.PreCommit{}, + v1_3.SectorDeal{}, + v1_3.SectorEvent{}, + v1_3.Sector{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Info{}, + v1_4.PreCommit{}, + v1_4.SectorDeal{}, + v1_4.SectorEvent{}, + v1_4.Sector{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Info{}, + v1_5.PreCommit{}, + v1_5.SectorDeal{}, + v1_5.SectorEvent{}, + v1_5.Sector{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Info{}, + v1_6.PreCommit{}, + v1_6.SectorDeal{}, + v1_6.SectorEvent{}, + v1_6.Sector{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Info{}, + v1_7.PreCommit{}, + v1_7.SectorDeal{}, + v1_7.SectorEvent{}, + v1_7.Sector{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Info{}, + v1_8.PreCommit{}, + v1_8.SectorDeal{}, + v1_8.SectorEvent{}, + v1_8.Sector{}, + }, nil + case actortypes.Version9: + return []Transformer{ + v1_9.Info{}, + v1_9.PreCommit{}, + v1_9.SectorDeal{}, + v1_9.SectorEvent{}, + v1_9.Sector{}, + }, nil + + } + return nil, fmt.Errorf("unsupported actor version for miner transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/miner/v1/types/types.go b/pkg/transform/timescale/actors/miner/v1/types/types.go new file mode 100644 index 000000000..7e1c57b07 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/types/types.go @@ -0,0 +1,12 @@ +package types + +import ( + "github.com/filecoin-project/go-address" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" +) + +type MinerStateChange struct { + Address address.Address + StateChange *minerdiff.StateDiffResult +} diff --git a/pkg/transform/timescale/actors/miner/v1/v0/info.go b/pkg/transform/timescale/actors/miner/v1/v0/info.go new file mode 100644 index 000000000..741e2aa4b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v0/info.go @@ -0,0 +1,97 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v0/precommits.go b/pkg/transform/timescale/actors/miner/v1/v0/precommits.go new file mode 100644 index 000000000..ecfe72a87 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v0/precommits.go @@ -0,0 +1,70 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel + +} diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go new file mode 100644 index 000000000..cfc670ec8 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go @@ -0,0 +1,69 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go new file mode 100644 index 000000000..f5d04fc44 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go @@ -0,0 +1,153 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sectors.go b/pkg/transform/timescale/actors/miner/v1/v0/sectors.go new file mode 100644 index 000000000..2b36a423b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v0/sectors.go @@ -0,0 +1,64 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v2/info.go b/pkg/transform/timescale/actors/miner/v1/v2/info.go new file mode 100644 index 000000000..a3db072fe --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v2/info.go @@ -0,0 +1,96 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v2/precommits.go b/pkg/transform/timescale/actors/miner/v1/v2/precommits.go new file mode 100644 index 000000000..5c1cbda50 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v2/precommits.go @@ -0,0 +1,69 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go new file mode 100644 index 000000000..7e2764209 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go @@ -0,0 +1,69 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go new file mode 100644 index 000000000..5a530baf9 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go @@ -0,0 +1,153 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sectors.go b/pkg/transform/timescale/actors/miner/v1/v2/sectors.go new file mode 100644 index 000000000..e5975bcb6 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v2/sectors.go @@ -0,0 +1,64 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v3/info.go b/pkg/transform/timescale/actors/miner/v1/v3/info.go new file mode 100644 index 000000000..029dcd161 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v3/info.go @@ -0,0 +1,96 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v3/precommits.go b/pkg/transform/timescale/actors/miner/v1/v3/precommits.go new file mode 100644 index 000000000..f1d806015 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v3/precommits.go @@ -0,0 +1,69 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go new file mode 100644 index 000000000..9b9f85ee6 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go @@ -0,0 +1,69 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go new file mode 100644 index 000000000..eeee246df --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go @@ -0,0 +1,153 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sectors.go b/pkg/transform/timescale/actors/miner/v1/v3/sectors.go new file mode 100644 index 000000000..28f39533e --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v3/sectors.go @@ -0,0 +1,64 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v4/info.go b/pkg/transform/timescale/actors/miner/v1/v4/info.go new file mode 100644 index 000000000..4fa4fbcc8 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v4/info.go @@ -0,0 +1,96 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v4/precommits.go b/pkg/transform/timescale/actors/miner/v1/v4/precommits.go new file mode 100644 index 000000000..deb4dbc2e --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v4/precommits.go @@ -0,0 +1,69 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go new file mode 100644 index 000000000..ac4f22208 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go @@ -0,0 +1,69 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go new file mode 100644 index 000000000..78aff5638 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go @@ -0,0 +1,153 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sectors.go b/pkg/transform/timescale/actors/miner/v1/v4/sectors.go new file mode 100644 index 000000000..ff9319c5b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v4/sectors.go @@ -0,0 +1,64 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v5/info.go b/pkg/transform/timescale/actors/miner/v1/v5/info.go new file mode 100644 index 000000000..5680f54ff --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v5/info.go @@ -0,0 +1,96 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v5/precommits.go b/pkg/transform/timescale/actors/miner/v1/v5/precommits.go new file mode 100644 index 000000000..3ef9e30e7 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v5/precommits.go @@ -0,0 +1,70 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel + +} diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go new file mode 100644 index 000000000..dea59441c --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go @@ -0,0 +1,69 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go new file mode 100644 index 000000000..b6efa12af --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go @@ -0,0 +1,153 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sectors.go b/pkg/transform/timescale/actors/miner/v1/v5/sectors.go new file mode 100644 index 000000000..a9fd0d7cc --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v5/sectors.go @@ -0,0 +1,64 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v6/info.go b/pkg/transform/timescale/actors/miner/v1/v6/info.go new file mode 100644 index 000000000..e3eb10e65 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v6/info.go @@ -0,0 +1,96 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v6/precommits.go b/pkg/transform/timescale/actors/miner/v1/v6/precommits.go new file mode 100644 index 000000000..4ae93453b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v6/precommits.go @@ -0,0 +1,70 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel + +} diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go new file mode 100644 index 000000000..136c6a2fa --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go @@ -0,0 +1,69 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go new file mode 100644 index 000000000..c4ea8f177 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go @@ -0,0 +1,153 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sectors.go b/pkg/transform/timescale/actors/miner/v1/v6/sectors.go new file mode 100644 index 000000000..b9952425f --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v6/sectors.go @@ -0,0 +1,64 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v7/info.go b/pkg/transform/timescale/actors/miner/v1/v7/info.go new file mode 100644 index 000000000..48a34b213 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v7/info.go @@ -0,0 +1,96 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v7/precommits.go b/pkg/transform/timescale/actors/miner/v1/v7/precommits.go new file mode 100644 index 000000000..9d21eb9de --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v7/precommits.go @@ -0,0 +1,69 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go new file mode 100644 index 000000000..1112c71db --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go @@ -0,0 +1,69 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go new file mode 100644 index 000000000..7cb5af61f --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go @@ -0,0 +1,153 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sectors.go b/pkg/transform/timescale/actors/miner/v1/v7/sectors.go new file mode 100644 index 000000000..8fb67b2fa --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v7/sectors.go @@ -0,0 +1,69 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV7, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) + for i, sector := range sectors { + sectorKeyCID := "" + if sector.SectorKeyCID != nil { + sectorKeyCID = sector.SectorKeyCID.String() + } + sectorModel[i] = &minermodel.MinerSectorInfoV7{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + SectorID: uint64(sector.SectorNumber), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + SectorKeyCID: sectorKeyCID, + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v8/info.go b/pkg/transform/timescale/actors/miner/v1/v8/info.go new file mode 100644 index 000000000..3507616a4 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v8/info.go @@ -0,0 +1,96 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v8/precommits.go b/pkg/transform/timescale/actors/miner/v1/v8/precommits.go new file mode 100644 index 000000000..0eee24323 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v8/precommits.go @@ -0,0 +1,70 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel + +} diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go new file mode 100644 index 000000000..5c6893f0a --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go @@ -0,0 +1,69 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go new file mode 100644 index 000000000..e30d1d6c8 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go @@ -0,0 +1,153 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sectors.go b/pkg/transform/timescale/actors/miner/v1/v8/sectors.go new file mode 100644 index 000000000..f93cd8d1c --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v8/sectors.go @@ -0,0 +1,69 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV7, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) + for i, sector := range sectors { + sectorKeyCID := "" + if sector.SectorKeyCID != nil { + sectorKeyCID = sector.SectorKeyCID.String() + } + sectorModel[i] = &minermodel.MinerSectorInfoV7{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + SectorID: uint64(sector.SectorNumber), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + SectorKeyCID: sectorKeyCID, + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/miner/v1/v9/info.go b/pkg/transform/timescale/actors/miner/v1/v9/info.go new file mode 100644 index 000000000..5d35a6179 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v9/info.go @@ -0,0 +1,96 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" +) + +type Info struct{} + +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) + } + return report.Finish() +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v1/v9/precommits.go b/pkg/transform/timescale/actors/miner/v1/v9/precommits.go new file mode 100644 index 000000000..dc6862349 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v9/precommits.go @@ -0,0 +1,70 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" +) + +type PreCommit struct{} + +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) + } + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) + } + return report.Finish() +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { + preCommitModel := make(minermodel.MinerPreCommitInfoV9List, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + unSealedCID := "" + if preCommit.Info.UnsealedCid != nil { + unSealedCID = preCommit.Info.UnsealedCid.String() + } + preCommitModel[i] = &minermodel.MinerPreCommitInfoV9{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + DealIDS: deals, + UnsealedCID: unSealedCID, + } + } + + return preCommitModel + +} diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go new file mode 100644 index 000000000..c896bdc6b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go @@ -0,0 +1,69 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go new file mode 100644 index 000000000..0da5cbd6a --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go @@ -0,0 +1,153 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + "github.com/filecoin-project/go-state-types/abi" + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sectors.go b/pkg/transform/timescale/actors/miner/v1/v9/sectors.go new file mode 100644 index 000000000..8a7898939 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/v9/sectors.go @@ -0,0 +1,69 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" +) + +type Sector struct{} + +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV7, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) + } + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) + } + return report.Finish() +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { + sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) + for i, sector := range sectors { + sectorKeyCID := "" + if sector.SectorKeyCID != nil { + sectorKeyCID = sector.SectorKeyCID.String() + } + sectorModel[i] = &minermodel.MinerSectorInfoV7{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + SectorID: uint64(sector.SectorNumber), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + SectorKeyCID: sectorKeyCID, + } + } + + return sectorModel +} diff --git a/pkg/transform/timescale/actors/power/router.go b/pkg/transform/timescale/actors/power/router.go new file mode 100644 index 000000000..1e2322cf3 --- /dev/null +++ b/pkg/transform/timescale/actors/power/router.go @@ -0,0 +1,41 @@ +package power + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +func TransformPowerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + return model.PersistableList{ + data.StartProcessingReport(tasktype.PowerActorClaim, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8, + actortypes.Version9: + return v1.TransformPowerState(ctx, s, version, current, executed, *root) + } + return nil, fmt.Errorf("unsupported version : %d", version) +} diff --git a/pkg/transform/timescale/actors/power/v1/router.go b/pkg/transform/timescale/actors/power/v1/router.go new file mode 100644 index 000000000..d030dcc71 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/router.go @@ -0,0 +1,90 @@ +package v1 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v9" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *powerdiff.StateDiffResult) model.Persistable +} + +func TransformPowerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + tramsformers, err := LookupPowerStateTransformers(version) + if err != nil { + return nil, err + } + powerState := new(powerdiff.StateChange) + if err := s.Get(ctx, root, powerState); err != nil { + return nil, err + } + powerStateDiff, err := powerState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range tramsformers { + m := t.Transform(ctx, current, executed, powerStateDiff) + out = append(out, m) + } + return out, nil +} + +func LookupPowerStateTransformers(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Claims{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Claims{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Claims{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Claims{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Claims{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Claims{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Claims{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Claims{}, + }, nil + case actortypes.Version9: + return []Transformer{ + v1_9.Claims{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for power transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/power/v1/v0/claims.go b/pkg/transform/timescale/actors/power/v1/v0/claims.go new file mode 100644 index 000000000..fc6a4570f --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v0/claims.go @@ -0,0 +1,47 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v2/claims.go b/pkg/transform/timescale/actors/power/v1/v2/claims.go new file mode 100644 index 000000000..6faf16d58 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v2/claims.go @@ -0,0 +1,47 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v3/claims.go b/pkg/transform/timescale/actors/power/v1/v3/claims.go new file mode 100644 index 000000000..8506044d4 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v3/claims.go @@ -0,0 +1,47 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/power" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v4/claims.go b/pkg/transform/timescale/actors/power/v1/v4/claims.go new file mode 100644 index 000000000..3cbe6f006 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v4/claims.go @@ -0,0 +1,47 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/power" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v5/claims.go b/pkg/transform/timescale/actors/power/v1/v5/claims.go new file mode 100644 index 000000000..a0b95b105 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v5/claims.go @@ -0,0 +1,47 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/power" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v6/claims.go b/pkg/transform/timescale/actors/power/v1/v6/claims.go new file mode 100644 index 000000000..c1333a143 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v6/claims.go @@ -0,0 +1,47 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/power" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v7/claims.go b/pkg/transform/timescale/actors/power/v1/v7/claims.go new file mode 100644 index 000000000..ff4d8fbfb --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v7/claims.go @@ -0,0 +1,47 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/power" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v8/claims.go b/pkg/transform/timescale/actors/power/v1/v8/claims.go new file mode 100644 index 000000000..650fbec05 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v8/claims.go @@ -0,0 +1,47 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v8/power" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/power/v1/v9/claims.go b/pkg/transform/timescale/actors/power/v1/v9/claims.go new file mode 100644 index 000000000..678ea9a35 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v9/claims.go @@ -0,0 +1,47 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v9/power" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + report.AddError(err) + continue + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + continue + } + report.AddModels(&powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/raw/state.go b/pkg/transform/timescale/actors/raw/state.go new file mode 100644 index 000000000..ed5e878f7 --- /dev/null +++ b/pkg/transform/timescale/actors/raw/state.go @@ -0,0 +1,120 @@ +package raw + +import ( + "context" + "encoding/json" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/lens/util" + "github.com/filecoin-project/lily/model" + commonmodel "github.com/filecoin-project/lily/model/actors/common" + visormodel "github.com/filecoin-project/lily/model/visor" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +type rawActorStateChange struct { + Address address.Address + StateChange *rawdiff.ActorChange +} + +func TransformActorStates(ctx context.Context, s store.Store, current, executed *types.TipSet, actorMapRoot *cid.Cid) (model.Persistable, error) { + if actorMapRoot == nil { + return model.PersistableList{ + data.StartProcessingReport(tasktype.Actor, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.ActorState, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + var out = model.PersistableList{} + actorMap, err := adt.AsMap(s, *actorMapRoot) + if err != nil { + return nil, err + } + + var actorChanges []*rawActorStateChange + actorState := new(rawdiff.ActorChange) + if err := actorMap.ForEach(actorState, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + val := new(rawdiff.ActorChange) + *val = *actorState + actorChanges = append(actorChanges, &rawActorStateChange{ + Address: addr, + StateChange: val, + }) + return nil + }); err != nil { + return nil, err + } + + out = append(out, ActorStateHandler(ctx, current, executed, actorChanges)) + + out = append(out, ActorHandler(ctx, current, executed, actorChanges)) + + return out, nil +} + +func ActorStateHandler(ctx context.Context, current, executed *types.TipSet, actors []*rawActorStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.ActorState, current) + // noop when actor is removed + for _, actor := range actors { + if actor.StateChange.Change == core.ChangeTypeRemove { + continue + } + + stateDump, err := vm.DumpActorState(util.ActorRegistry, actor.StateChange.Actor, actor.StateChange.Current) + if err != nil { + report.AddError(err) + continue + } + + state, err := json.Marshal(stateDump) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&commonmodel.ActorState{ + Height: int64(current.Height()), + Head: actor.StateChange.Actor.Head.String(), + Code: actor.StateChange.Actor.Code.String(), + State: string(state), + }) + } + return report.Finish() +} + +func ActorHandler(ctx context.Context, current, executed *types.TipSet, actors []*rawActorStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.Actor, current) + for _, actor := range actors { + if actor.StateChange.Change == core.ChangeTypeRemove { + continue + } + + report.AddModels(&commonmodel.Actor{ + Height: int64(current.Height()), + ID: actor.Address.String(), + StateRoot: current.ParentState().String(), + Code: actor.StateChange.Actor.Code.String(), + Head: actor.StateChange.Actor.Head.String(), + Balance: actor.StateChange.Actor.Balance.String(), + Nonce: actor.StateChange.Actor.Nonce, + }) + } + return report.Finish() +} diff --git a/pkg/transform/timescale/actors/verifreg/router.go b/pkg/transform/timescale/actors/verifreg/router.go new file mode 100644 index 000000000..2d3f782b8 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/router.go @@ -0,0 +1,65 @@ +package verifreg + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1" + v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v2" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + if version < actortypes.Version9 { + return model.PersistableList{ + data.StartProcessingReport(tasktype.VerifiedRegistryVerifiedClient, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.VerifiedRegistryVerifier, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + return model.PersistableList{ + data.StartProcessingReport("verified_registry_claims", current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport("verified_registry_allocation", current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.VerifiedRegistryVerifier, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8: + return v1.TransformVerifregState(ctx, s, version, current, executed, *root) + + case actortypes.Version9: + return v2.TransformVerifregState(ctx, s, version, current, executed, *root) + } + return nil, fmt.Errorf("unsupported version : %d", version) + +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/router.go b/pkg/transform/timescale/actors/verifreg/v1/router.go new file mode 100644 index 000000000..6a7ecc4ff --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/router.go @@ -0,0 +1,93 @@ +package v1 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v8" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + transformers, err := LookupVerifregStateTransformer(version) + verifregState := new(verifregdiff.StateChange) + if err := s.Get(ctx, root, verifregState); err != nil { + return nil, err + } + verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, verifrefStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} + +func LookupVerifregStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Clients{}, + v1_0.Verifiers{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Clients{}, + v1_2.Verifiers{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Clients{}, + v1_3.Verifiers{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Clients{}, + v1_4.Verifiers{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Clients{}, + v1_5.Verifiers{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Clients{}, + v1_6.Verifiers{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Clients{}, + v1_7.Verifiers{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Clients{}, + v1_8.Verifiers{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for verifreg transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v0/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v0/clients.go new file mode 100644 index 000000000..fbcde6557 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v0/clients.go @@ -0,0 +1,62 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v0/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v0/verifiers.go new file mode 100644 index 000000000..c747a726e --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v0/verifiers.go @@ -0,0 +1,62 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v2/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v2/clients.go new file mode 100644 index 000000000..9295043e8 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v2/clients.go @@ -0,0 +1,62 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v2/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v2/verifiers.go new file mode 100644 index 000000000..5daf24325 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v2/verifiers.go @@ -0,0 +1,62 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v3/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v3/clients.go new file mode 100644 index 000000000..2984ee80c --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v3/clients.go @@ -0,0 +1,62 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v3/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v3/verifiers.go new file mode 100644 index 000000000..e23b97c65 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v3/verifiers.go @@ -0,0 +1,62 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v4/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v4/clients.go new file mode 100644 index 000000000..734b4d6ad --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v4/clients.go @@ -0,0 +1,62 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v4/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v4/verifiers.go new file mode 100644 index 000000000..cc3f371f7 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v4/verifiers.go @@ -0,0 +1,62 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v5/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v5/clients.go new file mode 100644 index 000000000..c8beba8ee --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v5/clients.go @@ -0,0 +1,62 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v5/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v5/verifiers.go new file mode 100644 index 000000000..401845f74 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v5/verifiers.go @@ -0,0 +1,62 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v6/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v6/clients.go new file mode 100644 index 000000000..41017dca6 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v6/clients.go @@ -0,0 +1,62 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v6/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v6/verifiers.go new file mode 100644 index 000000000..456879a97 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v6/verifiers.go @@ -0,0 +1,62 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v7/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v7/clients.go new file mode 100644 index 000000000..c3ba6088d --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v7/clients.go @@ -0,0 +1,62 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v7/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v7/verifiers.go new file mode 100644 index 000000000..4e5341784 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v7/verifiers.go @@ -0,0 +1,62 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v8/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v8/clients.go new file mode 100644 index 000000000..e78844829 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v8/clients.go @@ -0,0 +1,62 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v8/verifreg" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v8/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v8/verifiers.go new file mode 100644 index 000000000..1b939e195 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v8/verifiers.go @@ -0,0 +1,62 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v8/verifreg" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v2/router.go b/pkg/transform/timescale/actors/verifreg/v2/router.go new file mode 100644 index 000000000..171e1c9d9 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v2/router.go @@ -0,0 +1,52 @@ +package v2 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" + v2_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v2/v9" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { + transformers, err := LookupVerifregStateTransformer(version) + verifregState := new(verifregdiff.StateChange) + if err := s.Get(ctx, root, verifregState); err != nil { + return nil, err + } + verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, verifrefStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} + +func LookupVerifregStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version9: + return []Transformer{ + v2_9.Verifiers{}, + //v2_9.Claims{}, + //v2_9.Allocations{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for verifreg transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/verifreg/v2/v9/allocations.go b/pkg/transform/timescale/actors/verifreg/v2/v9/allocations.go new file mode 100644 index 000000000..8da2488aa --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v2/v9/allocations.go @@ -0,0 +1,16 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" +) + +type Allocations struct{} + +func (Allocations) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + panic("TODO") +} diff --git a/pkg/transform/timescale/actors/verifreg/v2/v9/claims.go b/pkg/transform/timescale/actors/verifreg/v2/v9/claims.go new file mode 100644 index 000000000..d9bfe4a23 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v2/v9/claims.go @@ -0,0 +1,16 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + panic("TODO") +} diff --git a/pkg/transform/timescale/actors/verifreg/v2/v9/verifiers.go b/pkg/transform/timescale/actors/verifreg/v2/v9/verifiers.go new file mode 100644 index 000000000..b62b73925 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v2/v9/verifiers.go @@ -0,0 +1,62 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/data/report.go b/pkg/transform/timescale/data/report.go new file mode 100644 index 000000000..21178c144 --- /dev/null +++ b/pkg/transform/timescale/data/report.go @@ -0,0 +1,64 @@ +package data + +import ( + "time" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" +) + +type ReportError struct { + Error string +} + +type ProcessingReportBuilder struct { + errors []*ReportError + models model.PersistableList + report *visormodel.ProcessingReport +} + +func (b *ProcessingReportBuilder) WithStatus(status string) *ProcessingReportBuilder { + b.report.Status = status + return b +} + +func (b *ProcessingReportBuilder) WithInformation(info string) *ProcessingReportBuilder { + b.report.StatusInformation = info + return b +} + +func (b *ProcessingReportBuilder) AddError(err error) *ProcessingReportBuilder { + b.errors = append(b.errors, &ReportError{Error: err.Error()}) + return b +} + +func (b *ProcessingReportBuilder) AddModels(m ...model.Persistable) *ProcessingReportBuilder { + b.models = append(b.models, m...) + return b +} + +func (b *ProcessingReportBuilder) Finish() model.PersistableList { + b.report.CompletedAt = time.Now() + if len(b.errors) == 0 { + if b.report.Status == "" { + b.WithStatus(visormodel.ProcessingStatusOK) + } + } else { + b.WithStatus(visormodel.ProcessingStatusError) + } + return model.PersistableList{b.models, b.report} +} + +func StartProcessingReport(task string, ts *types.TipSet) *ProcessingReportBuilder { + return &ProcessingReportBuilder{ + report: &visormodel.ProcessingReport{ + Height: int64(ts.Height()), + StateRoot: ts.ParentState().String(), + Reporter: "Deprecate", + Task: task, + StartedAt: time.Now(), + }, + } +} diff --git a/pkg/transform/timescale/fullblock/blocks.go b/pkg/transform/timescale/fullblock/blocks.go new file mode 100644 index 000000000..f8cd1ae21 --- /dev/null +++ b/pkg/transform/timescale/fullblock/blocks.go @@ -0,0 +1,80 @@ +package fullblock + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/model/blocks" + messagemodel "github.com/filecoin-project/lily/model/messages" + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +func mustMakeTipsetFromFullBlocks(fullBlocks map[cid.Cid]*chain.FullBlock) *types.TipSet { + var header []*types.BlockHeader + for _, fb := range fullBlocks { + header = append(header, fb.Block) + } + ts, err := types.NewTipSet(header) + if err != nil { + panic(err) + } + return ts +} + +func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.BlockHeader, mustMakeTipsetFromFullBlocks(fullBlocks)) + for _, fb := range fullBlocks { + report.AddModels(&blocks.BlockHeader{ + Height: int64(fb.Block.Height), + Cid: fb.Block.Cid().String(), + Miner: fb.Block.Miner.String(), + ParentWeight: fb.Block.ParentWeight.String(), + ParentBaseFee: fb.Block.ParentBaseFee.String(), + ParentStateRoot: fb.Block.ParentStateRoot.String(), + WinCount: fb.Block.ElectionProof.WinCount, + Timestamp: fb.Block.Timestamp, + ForkSignaling: fb.Block.ForkSignaling, + }) + } + return report.Finish() +} + +func ExtractBlockParents(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.BlockParent, mustMakeTipsetFromFullBlocks(fullBlocks)) + for _, fb := range fullBlocks { + for _, p := range fb.Block.Parents { + report.AddModels(&blocks.BlockParent{ + Height: int64(fb.Block.Height), + Block: fb.Block.Cid().String(), + Parent: p.String(), + }) + } + } + return report.Finish() +} + +func ExtractBlockMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.BlockMessage, mustMakeTipsetFromFullBlocks(fullBlocks)) + for _, fb := range fullBlocks { + for _, msg := range fb.BlsMessages { + report.AddModels(&messagemodel.BlockMessage{ + Height: int64(fb.Block.Height), + Block: fb.Block.Cid().String(), + Message: msg.Message.Cid().String(), + }) + } + for _, msg := range fb.SecpMessages { + report.AddModels(&messagemodel.BlockMessage{ + Height: int64(fb.Block.Height), + Block: fb.Block.Cid().String(), + Message: msg.Message.Cid().String(), + }) + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/fullblock/messages.go b/pkg/transform/timescale/fullblock/messages.go new file mode 100644 index 000000000..b5f263547 --- /dev/null +++ b/pkg/transform/timescale/fullblock/messages.go @@ -0,0 +1,95 @@ +package fullblock + +import ( + "context" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/indexer/tasktype" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/model/messages" + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" +) + +func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.Message, mustMakeTipsetFromFullBlocks(fullBlocks)) + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + report.AddModels(&messages.Message{ + Height: int64(fb.Block.Height), + Cid: msg.Message.Cid().String(), + From: msg.Message.Message.From.String(), + To: msg.Message.Message.To.String(), + Value: msg.Message.Message.Value.String(), + GasFeeCap: msg.Message.Message.GasFeeCap.String(), + GasPremium: msg.Message.Message.GasPremium.String(), + GasLimit: msg.Message.Message.GasLimit, + SizeBytes: msg.Message.ChainLength(), + Nonce: msg.Message.Message.Nonce, + Method: uint64(msg.Message.Message.Method), + }) + } + for _, msg := range fb.BlsMessages { + report.AddModels(&messages.Message{ + Height: int64(fb.Block.Height), + Cid: msg.Message.Cid().String(), + From: msg.Message.From.String(), + To: msg.Message.To.String(), + Value: msg.Message.Value.String(), + GasFeeCap: msg.Message.GasFeeCap.String(), + GasPremium: msg.Message.GasPremium.String(), + GasLimit: msg.Message.GasLimit, + SizeBytes: msg.Message.ChainLength(), + Nonce: msg.Message.Nonce, + Method: uint64(msg.Message.Method), + }) + } + } + return report.Finish() +} + +func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.VMMessage, mustMakeTipsetFromFullBlocks(fullBlocks)) + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + for _, vm := range msg.VmMessages { + report.AddModels(&messages.VMMessage{ + Height: int64(fb.Block.Height), + StateRoot: fb.Block.ParentStateRoot.String(), + Cid: vm.Message.Cid().String(), + Source: vm.Source.String(), + From: vm.Message.From.String(), + To: vm.Message.To.String(), + Value: vm.Message.Value.String(), + Method: uint64(vm.Message.Method), + ActorCode: "Deprecate", + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Params: "TODO make bytes", //vm.Message.Params + Returns: "TODO make bytes", //vm.Receipt.Return + }) + } + } + for _, msg := range fb.BlsMessages { + for _, vm := range msg.VmMessages { + report.AddModels(&messages.VMMessage{ + Height: int64(fb.Block.Height), + StateRoot: fb.Block.ParentStateRoot.String(), + Cid: vm.Message.Cid().String(), + Source: vm.Source.String(), + From: vm.Message.From.String(), + To: vm.Message.To.String(), + Value: vm.Message.Value.String(), + Method: uint64(vm.Message.Method), + ActorCode: "Deprecate", + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Params: "TODO make bytes", //vm.Message.Params + Returns: "TODO make bytes", //vm.Receipt.Return + }) + } + } + } + return report.Finish() +} diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go new file mode 100644 index 000000000..9b612c278 --- /dev/null +++ b/pkg/transform/timescale/processor.go @@ -0,0 +1,160 @@ +package timescale + +import ( + "context" + "fmt" + "io" + + "github.com/filecoin-project/go-state-types/abi" + actorstypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + v1car "github.com/ipld/go-car" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" + init_ "github.com/filecoin-project/lily/pkg/transform/timescale/actors/init" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/raw" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg" + "github.com/filecoin-project/lily/pkg/transform/timescale/fullblock" +) + +var log = logging.Logger("/lily/transform/timescale") + +type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version + +func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVersionGetter) error { + bs := blockstore.NewMemorySync() + header, err := v1car.LoadCar(ctx, bs, r) + if err != nil { + return err + } + if len(header.Roots) != 1 { + return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) + } + + adtStore := store.WrapBlockStore(ctx, bs) + + rootIPLDContainer := new(cbor.RootStateIPLD) + if err := adtStore.Get(ctx, header.Roots[0], rootIPLDContainer); err != nil { + return err + } + log.Infow("open root", "car_root", header.Roots[0], zap.Inline(rootIPLDContainer)) + + stateExtractionIPLDContainer := new(cbor.StateExtractionIPLD) + if err := adtStore.Get(ctx, rootIPLDContainer.State, stateExtractionIPLDContainer); err != nil { + return err + } + log.Infow("open state extraction", zap.Inline(stateExtractionIPLDContainer)) + current := &stateExtractionIPLDContainer.Current + parent := &stateExtractionIPLDContainer.Parent + + av, err := core.ActorVersionForTipSet(ctx, current, nvg) + if err != nil { + return err + } + + toStorage := model.PersistableList{} + fbModels, err := HandleFullBlocks(ctx, adtStore, current, parent, stateExtractionIPLDContainer.FullBlocks) + if err != nil { + return err + } + + toStorage = append(toStorage, fbModels) + + _, err = HandleImplicitMessages(ctx, adtStore, current, parent, stateExtractionIPLDContainer.ImplicitMessages) + if err != nil { + return err + } + + actorModels, err := HandleActorStateChanges(ctx, adtStore, current, parent, av, stateExtractionIPLDContainer.Actors) + if err != nil { + return err + } + + toStorage = append(toStorage, actorModels) + + return strg.PersistBatch(ctx, toStorage) +} + +func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { + actorIPLDContainer := new(actors.ActorStateChangesIPLD) + if err := s.Get(ctx, root, actorIPLDContainer); err != nil { + return nil, err + } + log.Infow("open actor state changes", zap.Inline(actorIPLDContainer)) + + out := model.PersistableList{} + marketModels, err := market.TransformMarketState(ctx, s, av, current, parent, actorIPLDContainer.MarketActor) + if err != nil { + return nil, err + } + out = append(out, marketModels) + + minerModels, err := miner.TransformMinerState(ctx, s, av, current, parent, actorIPLDContainer.MinerActors) + if err != nil { + return nil, err + } + out = append(out, minerModels) + + powerModels, err := power.TransformPowerState(ctx, s, av, current, parent, actorIPLDContainer.PowerActor) + if err != nil { + return nil, err + } + out = append(out, powerModels) + + initModels, err := init_.TransformInitState(ctx, s, current, parent, actorIPLDContainer.InitActor) + if err != nil { + return nil, err + } + out = append(out, initModels) + + verifregModels, err := verifreg.TransformVerifregState(ctx, s, av, current, parent, actorIPLDContainer.VerifregActor) + if err != nil { + return nil, err + } + out = append(out, verifregModels) + + rawModels, err := raw.TransformActorStates(ctx, s, current, parent, actorIPLDContainer.RawActors) + if err != nil { + return nil, err + } + out = append(out, rawModels) + + return out, nil +} + +func HandleFullBlocks(ctx context.Context, s store.Store, current, parent *types.TipSet, root cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + fullBlockMap, err := messages.DecodeFullBlockHAMT(ctx, s, root) + if err != nil { + return nil, err + } + out = append(out, fullblock.ExtractBlockHeaders(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractBlockParents(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractBlockMessages(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractMessages(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractVmMessages(ctx, fullBlockMap)) + + return out, nil +} + +func HandleImplicitMessages(ctx context.Context, s store.Store, current, parent *types.TipSet, root cid.Cid) (interface{}, error) { + implicitMessages, err := messages.DecodeImplicitMessagesHAMT(ctx, s, root) + if err != nil { + return nil, err + } + _ = implicitMessages + return nil, nil +} diff --git a/pkg/util/cid.go b/pkg/util/cid.go new file mode 100644 index 000000000..25a4c14bd --- /dev/null +++ b/pkg/util/cid.go @@ -0,0 +1,34 @@ +package util + +import ( + "bytes" + + "github.com/filecoin-project/go-state-types/abi" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" +) + +func CidOf(v typegen.CBORMarshaler) (cid.Cid, error) { + buf := new(bytes.Buffer) + if err := v.MarshalCBOR(buf); err != nil { + return cid.Undef, err + } + c, err := abi.CidBuilder.Sum(buf.Bytes()) + if err != nil { + return cid.Undef, err + } + b, err := block.NewBlockWithCid(buf.Bytes(), c) + if err != nil { + return cid.Undef, err + } + return b.Cid(), nil +} + +func MustCidOf(v typegen.CBORMarshaler) cid.Cid { + c, err := CidOf(v) + if err != nil { + panic(err) + } + return c +} diff --git a/schemas/v1/base.go b/schemas/v1/base.go index 4845d3cf9..f8e61fffe 100644 --- a/schemas/v1/base.go +++ b/schemas/v1/base.go @@ -596,7 +596,7 @@ COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.prov COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.start_epoch IS 'The epoch at which this deal with begin. Storage deal must appear in a sealed (proven) sector no later than start_epoch, otherwise it is invalid.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.end_epoch IS 'The epoch at which this deal with end.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.storage_price_per_epoch IS 'The amount of FIL (in attoFIL) that will be transferred from the client to the provider every epoch this deal is active for.'; -COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.provider_collateral IS 'The amount of FIL (in attoFIL) the provider has pledged as collateral. The Provider deal collateral is only slashed when a sector is terminated before the deal expires.'; +COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.provider_collateral IS 'The amount of FIL (in attoFIL) the provider has pledged as collateral. The Client deal collateral is only slashed when a sector is terminated before the deal expires.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.client_collateral IS 'The amount of FIL (in attoFIL) the client has pledged as collateral.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.label IS 'An arbitrary client chosen label to apply to the deal.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.height IS 'Epoch at which this deal proposal was added or changed.'; diff --git a/shed/newschema/main.go b/shed/newschema/main.go new file mode 100644 index 000000000..f16e496b8 --- /dev/null +++ b/shed/newschema/main.go @@ -0,0 +1,315 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + v1car "github.com/ipld/go-car" + "github.com/urfave/cli/v2" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/schema" + + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" + "github.com/filecoin-project/lily/pkg/transform/gorm/models" + dbtypes "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +func main() { + app := &cli.App{ + Name: "transform", + Commands: []*cli.Command{ + TransformCmd, + }, + } + app.Setup() + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stdout, err.Error()) + os.Exit(1) + } +} + +var TransformCmd = &cli.Command{ + Name: "delta", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Usage: "Name of the file to transform", + }, + &cli.StringFlag{ + Name: "db", + Value: "host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable", + }, + }, + Action: func(cctx *cli.Context) error { + // Open the delta file (car file), this contains the filecoin state + f, err := os.OpenFile(cctx.String("filename"), os.O_RDONLY, 0o644) + defer f.Close() + if err != nil { + return err + } + + // connect to the database, were gonna write stuff to this. + db, err := gorm.Open(postgres.Open(cctx.String("db")), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list + // TODO figure out how to make a new schema with gorm... + }, + }) + if err != nil { + return err + } + + // extract the delta state to the database and exit. + return Run(cctx.Context, f, db) + }, +} + +func Run(ctx context.Context, r io.Reader, db *gorm.DB) error { + // create a blockstore and load the contents of the car file (reader is a pointer to the car file) into it. + bs := blockstore.NewMemorySync() + header, err := v1car.LoadCar(ctx, bs, r) + if err != nil { + return err + } + // expect to have a single root (cid) pointing to content + if len(header.Roots) != 1 { + return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) + } + + // we need to wrap the blockstore to meet some dumb interface. + s := store.WrapBlockStore(ctx, bs) + + // load the root container, this contains netwwork metadata and the root (cid) of the extracted state. + rootIPLDContainer := new(cbor.RootStateIPLD) + if err := s.Get(ctx, header.Roots[0], rootIPLDContainer); err != nil { + return err + } + + // load the extracted state, this contains links to fullblocks, implicit message, network economics, and actor states + stateExtractionIPLDContainer := new(cbor.StateExtractionIPLD) + if err := s.Get(ctx, rootIPLDContainer.State, stateExtractionIPLDContainer); err != nil { + return err + } + + // ohh and it also contains the tipset the extraction was performed over. + current := &stateExtractionIPLDContainer.Current + parent := &stateExtractionIPLDContainer.Parent + + // for now we will only handle the fullblock dag. + if err := HandleFullBlocks(ctx, db, s, current, parent, stateExtractionIPLDContainer.FullBlocks); err != nil { + return err + } + + return nil +} + +func HandleFullBlocks(ctx context.Context, db *gorm.DB, s store.Store, current, parent *types.TipSet, root cid.Cid) error { + // decode the content at the root (cid) into a concrete type we can inspect, a map of block CID to the block, its messages, their receipts and vm messages. + fullBlocks, err := messages.DecodeFullBlockHAMT(ctx, s, root) + if err != nil { + return err + } + // migrate the database, this creates new tables or updates existing ones with new fields + if err := db.AutoMigrate(&models.BlockHeaderModel{}, &models.Message{}, models.MessageReceipt{}, models.VmMessage{}, models.ParsedMessageParams{}); err != nil { + return err + } + // make some blockheaders and plop em in the database + if err := db.Create(MakeBlockHeaderModels(ctx, fullBlocks)).Error; err != nil { + return err + } + // now do messages + if err := db.Create(MakeMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + // finally receipts + if err := db.Create(MakeReceipts(ctx, fullBlocks)).Error; err != nil { + return err + } + + if err := db.Create(MakeVmMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + + // okay all done. + return nil +} + +func MakeBlockHeaderModels(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.BlockHeaderModel { + out := make([]*models.BlockHeaderModel, 0, len(fullBlocks)) + for _, fb := range fullBlocks { + out = append(out, &models.BlockHeaderModel{ + Cid: dbtypes.DbCID{CID: fb.Block.Cid()}, + StateRoot: dbtypes.DbCID{CID: fb.Block.ParentStateRoot}, + Height: int64(fb.Block.Height), + Miner: dbtypes.DbAddr{Addr: fb.Block.Miner}, + ParentWeight: dbtypes.DbBigInt{BigInt: fb.Block.ParentWeight}, + TimeStamp: fb.Block.Timestamp, + ForkSignaling: fb.Block.ForkSignaling, + BaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, + WinCount: fb.Block.ElectionProof.WinCount, + ParentBaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, + }) + } + return out +} + +func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.Message { + // messages can be contained in more than 1 block, this is used to prevent persisting them twice + seen := cid.NewSet() + var out []*models.Message + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + if !seen.Visit(smsg.Message.Cid()) { + continue + } + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: smsg.Message.Cid()}, + Version: int64(smsg.Message.Message.Version), + To: dbtypes.DbAddr{Addr: smsg.Message.Message.To}, + From: dbtypes.DbAddr{smsg.Message.Message.From}, + Nonce: smsg.Message.Message.Nonce, + Value: dbtypes.DbToken{Token: smsg.Message.Message.Value}, + GasLimit: smsg.Message.Message.GasLimit, + GasFeeCap: dbtypes.DbToken{Token: smsg.Message.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: smsg.Message.Message.GasPremium}, + Method: uint64(smsg.Message.Message.Method), + Params: smsg.Message.Message.Params, + }) + } + for _, msg := range fb.BlsMessages { + if !seen.Visit(msg.Message.Cid()) { + continue + } + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: msg.Message.Cid()}, + Version: int64(msg.Message.Version), + To: dbtypes.DbAddr{Addr: msg.Message.To}, + From: dbtypes.DbAddr{msg.Message.From}, + Nonce: msg.Message.Nonce, + Value: dbtypes.DbToken{Token: msg.Message.Value}, + GasLimit: msg.Message.GasLimit, + GasFeeCap: dbtypes.DbToken{Token: msg.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: msg.Message.GasPremium}, + Method: uint64(msg.Message.Method), + Params: msg.Message.Params, + }) + } + } + return out +} + +func MakeReceipts(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.MessageReceipt { + var out []*models.MessageReceipt + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + // TODO this is buggy + if smsg.Receipt.GasOutputs == nil { + continue + } + actorError := "" + if smsg.Receipt.ActorError != nil { + actorError = smsg.Receipt.ActorError.Error + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: smsg.Message.Cid()}, + Receipt: models.Receipt{ + Index: smsg.Receipt.Index, + ExitCode: int64(smsg.Receipt.Receipt.ExitCode), + GasUsed: smsg.Receipt.Receipt.GasUsed, + Return: smsg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.Refund}, + GasRefund: smsg.Receipt.GasOutputs.GasRefund, + GasBurned: smsg.Receipt.GasOutputs.GasBurned, + Error: actorError, + }) + } + for _, msg := range fb.BlsMessages { + if msg.Receipt.GasOutputs == nil { + continue + } + actorError := "" + if msg.Receipt.ActorError != nil { + actorError = msg.Receipt.ActorError.Error + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: msg.Message.Cid()}, + Receipt: models.Receipt{ + Index: msg.Receipt.Index, + ExitCode: int64(msg.Receipt.Receipt.ExitCode), + GasUsed: msg.Receipt.Receipt.GasUsed, + Return: msg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.Refund}, + GasRefund: msg.Receipt.GasOutputs.GasRefund, + GasBurned: msg.Receipt.GasOutputs.GasBurned, + Error: actorError, + }) + } + } + return out +} + +func MakeVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.VmMessage { + var out []*models.VmMessage + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + for _, msg := range fb.BlsMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + } + return out +} diff --git a/storage/sql.go b/storage/sql.go index 5aa1602cd..b314e1557 100644 --- a/storage/sql.go +++ b/storage/sql.go @@ -437,24 +437,27 @@ func (s *TxStorage) PersistModel(ctx context.Context, m interface{}) error { return nil } -// GenerateUpsertString accepts a lily model and returns two string containing SQL that may be used +// GenerateUpsertStrings accepts a lily model and returns two string containing SQL that may be used // to upsert the model. The first string is the conflict statement and the second is the insert. // // Example given the below model: // -// type SomeModel struct { -// Height int64 `pg:",pk,notnull,use_zero"` -// MinerID string `pg:",pk,notnull"` -// StateRoot string `pg:",pk,notnull"` -// OwnerID string `pg:",notnull"` -// WorkerID string `pg:",notnull"` -// } +// type SomeModel struct { +// Height int64 `pg:",pk,notnull,use_zero"` +// MinerID string `pg:",pk,notnull"` +// StateRoot string `pg:",pk,notnull"` +// OwnerID string `pg:",notnull"` +// WorkerID string `pg:",notnull"` +// } // // The strings returned are: // conflict string: +// // "(cid, height, state_root) DO UPDATE" +// // update string: -// "owner_id" = EXCLUDED.owner_id, "worker_id" = EXCLUDED.worker_id +// +// "owner_id" = EXCLUDED.owner_id, "worker_id" = EXCLUDED.worker_id func GenerateUpsertStrings(model interface{}) (string, string) { var cf []string var ucf []string diff --git a/tasks/actorstate/account/account.go b/tasks/actorstate/account/account.go index 6ac21a2ab..76564c176 100644 --- a/tasks/actorstate/account/account.go +++ b/tasks/actorstate/account/account.go @@ -16,7 +16,7 @@ type AccountExtractor struct{} // Extract will create persistable data for a given actor's state. func (AccountExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - _, span := otel.Tracer("").Start(ctx, "AccountExtractor.Extract") + _, span := otel.Tracer("").Start(ctx, "AccountExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/datacap/balance.go b/tasks/actorstate/datacap/balance.go index a4b91a545..390620581 100644 --- a/tasks/actorstate/datacap/balance.go +++ b/tasks/actorstate/datacap/balance.go @@ -23,7 +23,7 @@ type BalanceExtractor struct{} func (BalanceExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "BalanceExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "BalancesExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "BalancesExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/init_/init.go b/tasks/actorstate/init_/init.go index 5c8dadd74..29cde1fbe 100644 --- a/tasks/actorstate/init_/init.go +++ b/tasks/actorstate/init_/init.go @@ -24,7 +24,7 @@ type InitExtractor struct{} func (InitExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "InitExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "InitExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "InitExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/market/deal_proposal.go b/tasks/actorstate/market/deal_proposal.go index a12ee870f..2b44bb89a 100644 --- a/tasks/actorstate/market/deal_proposal.go +++ b/tasks/actorstate/market/deal_proposal.go @@ -24,7 +24,7 @@ type DealProposalExtractor struct{} func (DealProposalExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "DealProposalExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "DealProposalExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "DealProposalExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/market/deal_state.go b/tasks/actorstate/market/deal_state.go index 225c4769a..0939c8eb7 100644 --- a/tasks/actorstate/market/deal_state.go +++ b/tasks/actorstate/market/deal_state.go @@ -20,7 +20,7 @@ type DealStateExtractor struct{} func (DealStateExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "DealStateExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "DealStateExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "DealStateExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/beneficaries.go b/tasks/actorstate/miner/beneficaries.go index b0316b03d..bbaee38a4 100644 --- a/tasks/actorstate/miner/beneficaries.go +++ b/tasks/actorstate/miner/beneficaries.go @@ -17,7 +17,7 @@ type BeneficiaryExtractor struct{} func (BeneficiaryExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "BeneficiaryExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "BeneficiaryExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "BeneficiaryExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/deadline_info.go b/tasks/actorstate/miner/deadline_info.go index 76f884d0f..e87e2cda9 100644 --- a/tasks/actorstate/miner/deadline_info.go +++ b/tasks/actorstate/miner/deadline_info.go @@ -16,7 +16,7 @@ type DeadlineInfoExtractor struct{} func (DeadlineInfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "DeadlineInfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "DeadlineInfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "DeadlineInfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/fee_debt.go b/tasks/actorstate/miner/fee_debt.go index 0a7c9a25f..254e1bb33 100644 --- a/tasks/actorstate/miner/fee_debt.go +++ b/tasks/actorstate/miner/fee_debt.go @@ -16,7 +16,7 @@ type FeeDebtExtractor struct{} func (FeeDebtExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "FeeDebtExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "FeeDebtExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "FeeDebtExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/info.go b/tasks/actorstate/miner/info.go index f545b82ef..f58fcb37c 100644 --- a/tasks/actorstate/miner/info.go +++ b/tasks/actorstate/miner/info.go @@ -22,7 +22,7 @@ type InfoExtractor struct{} func (InfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "InfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "InfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "InfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/locked_funds.go b/tasks/actorstate/miner/locked_funds.go index 0e3c4d0f8..7acde81cd 100644 --- a/tasks/actorstate/miner/locked_funds.go +++ b/tasks/actorstate/miner/locked_funds.go @@ -16,7 +16,7 @@ type LockedFundsExtractor struct{} func (LockedFundsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "LockedFundsExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "LockedFundsExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "LockedFundsExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/post.go b/tasks/actorstate/miner/post.go index a891751fd..45f307be2 100644 --- a/tasks/actorstate/miner/post.go +++ b/tasks/actorstate/miner/post.go @@ -22,7 +22,7 @@ type PoStExtractor struct{} func (PoStExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "PoStExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "PoStExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "PoStExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/precommit.go b/tasks/actorstate/miner/precommit.go index 8c7d70dcc..85c381f73 100644 --- a/tasks/actorstate/miner/precommit.go +++ b/tasks/actorstate/miner/precommit.go @@ -18,7 +18,7 @@ type PreCommitInfoExtractorV8 struct{} func (PreCommitInfoExtractorV8) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "PreCommitInfoV8Extractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfo.Extract") + ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfo.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/precommitv9.go b/tasks/actorstate/miner/precommitv9.go index b558c950a..59c213afd 100644 --- a/tasks/actorstate/miner/precommitv9.go +++ b/tasks/actorstate/miner/precommitv9.go @@ -17,7 +17,7 @@ type PreCommitInfoExtractorV9 struct{} func (PreCommitInfoExtractorV9) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "PreCommitInfoV9Extractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfoV9.Extract") + ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfoV9.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sector.go b/tasks/actorstate/miner/sector.go index 267288611..567e4c693 100644 --- a/tasks/actorstate/miner/sector.go +++ b/tasks/actorstate/miner/sector.go @@ -17,7 +17,7 @@ type SectorInfoExtractor struct{} func (SectorInfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "SectorInfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "SectorInfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "SectorInfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sector_deals.go b/tasks/actorstate/miner/sector_deals.go index 32bd078fa..ea5773fe2 100644 --- a/tasks/actorstate/miner/sector_deals.go +++ b/tasks/actorstate/miner/sector_deals.go @@ -18,7 +18,7 @@ type SectorDealsExtractor struct{} func (SectorDealsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "SectorDealsExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "SectorDealsExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "SectorDealsExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sector_events.go b/tasks/actorstate/miner/sector_events.go index 28ab6fdef..ef4913a5c 100644 --- a/tasks/actorstate/miner/sector_events.go +++ b/tasks/actorstate/miner/sector_events.go @@ -23,7 +23,7 @@ type SectorEventsExtractor struct{} func (SectorEventsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "SectorEventsExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "SectorEventsExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "SectorEventsExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sectorv7.go b/tasks/actorstate/miner/sectorv7.go index c02eba91e..f2ba52250 100644 --- a/tasks/actorstate/miner/sectorv7.go +++ b/tasks/actorstate/miner/sectorv7.go @@ -17,7 +17,7 @@ type V7SectorInfoExtractor struct{} func (V7SectorInfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "V7SectorInfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "V7SectorInfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "V7SectorInfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/multisig/multisig.go b/tasks/actorstate/multisig/multisig.go index a739bcb65..87fca0e8e 100644 --- a/tasks/actorstate/multisig/multisig.go +++ b/tasks/actorstate/multisig/multisig.go @@ -23,7 +23,7 @@ type MultiSigActorExtractor struct{} func (MultiSigActorExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "MultiSigActorExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "MultiSigExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "MultiSigExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/power/chain_power.go b/tasks/actorstate/power/chain_power.go index c068f23be..cc992491f 100644 --- a/tasks/actorstate/power/chain_power.go +++ b/tasks/actorstate/power/chain_power.go @@ -20,7 +20,7 @@ type ChainPowerExtractor struct{} func (ChainPowerExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "ChainPowerExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "ChainPowerExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "ChainPowerExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/power/claimed_power.go b/tasks/actorstate/power/claimed_power.go index 5dc8810bf..6e2842768 100644 --- a/tasks/actorstate/power/claimed_power.go +++ b/tasks/actorstate/power/claimed_power.go @@ -19,7 +19,7 @@ type ClaimedPowerExtractor struct{} func (c ClaimedPowerExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "ClaimedPowerExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "ClaimedPowerExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "ClaimedPowerExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/power/power.go b/tasks/actorstate/power/power.go index 40ead37de..727ab33a0 100644 --- a/tasks/actorstate/power/power.go +++ b/tasks/actorstate/power/power.go @@ -68,7 +68,7 @@ func (p *PowerStateExtractionContext) HasPreviousState() bool { } func (StoragePowerExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - ctx, span := otel.Tracer("").Start(ctx, "StoragePowerExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "StoragePowerExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/raw/actor.go b/tasks/actorstate/raw/actor.go index 1eec97005..db0d72f6d 100644 --- a/tasks/actorstate/raw/actor.go +++ b/tasks/actorstate/raw/actor.go @@ -19,9 +19,9 @@ var log = logging.Logger("lily/tasks/rawactor") type RawActorExtractor struct{} func (RawActorExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - log.Debugw("Extract", zap.String("extractor", "RawActorExtractor"), zap.Inline(a)) + log.Debugw("Transform", zap.String("extractor", "RawActorExtractor"), zap.Inline(a)) - _, span := otel.Tracer("").Start(ctx, "RawActorExtractor.Extract") + _, span := otel.Tracer("").Start(ctx, "RawActorExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/raw/actor_state.go b/tasks/actorstate/raw/actor_state.go index c599f8759..7319de31b 100644 --- a/tasks/actorstate/raw/actor_state.go +++ b/tasks/actorstate/raw/actor_state.go @@ -16,8 +16,8 @@ import ( type RawActorStateExtractor struct{} func (RawActorStateExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - log.Debugw("Extract", zap.String("extractor", "RawStateActorExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "RawActorStateExtractor.Extract") + log.Debugw("Transform", zap.String("extractor", "RawStateActorExtractor"), zap.Inline(a)) + ctx, span := otel.Tracer("").Start(ctx, "RawActorStateExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/reward/reward.go b/tasks/actorstate/reward/reward.go index f32519f44..2ed53af01 100644 --- a/tasks/actorstate/reward/reward.go +++ b/tasks/actorstate/reward/reward.go @@ -21,7 +21,7 @@ type RewardExtractor struct{} func (RewardExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "RewardExtractor"), zap.Inline(a)) - _, span := otel.Tracer("").Start(ctx, "RewardExtractor.Extract") + _, span := otel.Tracer("").Start(ctx, "RewardExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/verifreg/client.go b/tasks/actorstate/verifreg/client.go index c958f7fbd..9c8075f08 100644 --- a/tasks/actorstate/verifreg/client.go +++ b/tasks/actorstate/verifreg/client.go @@ -19,7 +19,7 @@ type ClientExtractor struct{} func (ClientExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "ClientExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryClientExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryClientExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/verifreg/verifier.go b/tasks/actorstate/verifreg/verifier.go index a7ef3b477..93e6f2413 100644 --- a/tasks/actorstate/verifreg/verifier.go +++ b/tasks/actorstate/verifreg/verifier.go @@ -22,7 +22,7 @@ type VerifierExtractor struct{} func (VerifierExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "VerifierExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "VerifierExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "VerifierExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/verifreg/verifreg.go b/tasks/actorstate/verifreg/verifreg.go index e16dd7f8c..022cb5eda 100644 --- a/tasks/actorstate/verifreg/verifreg.go +++ b/tasks/actorstate/verifreg/verifreg.go @@ -66,7 +66,7 @@ func NewVerifiedRegistryExtractorContext(ctx context.Context, a actorstate.Actor } func (VerifiedRegistryExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/api.go b/tasks/api.go index a0f78c956..e8da0ddcf 100644 --- a/tasks/api.go +++ b/tasks/api.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/chain/actors/builtin/miner" @@ -52,6 +53,7 @@ type DataSource interface { MinerPower(ctx context.Context, addr address.Address, ts *types.TipSet) (*api.MinerPower, error) ActorStateChanges(ctx context.Context, ts, pts *types.TipSet) (ActorStateChangeDiff, error) MessageExecutions(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecution, error) + MessageExecutionsV2(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecutionV2, error) Store() adt.Store ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) @@ -66,4 +68,9 @@ type DataSource interface { MinerLoad(store adt.Store, act *types.Actor) (miner.State, error) ShouldBurnFn(ctx context.Context, ts *types.TipSet) (lens.ShouldBurnFn, error) + + ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) + + NetworkName(ctx context.Context) (string, error) + NetworkVersion(ctx context.Context, tsk types.TipSetKey) (uint, error) } diff --git a/tasks/messages/message_test.go b/tasks/messages/message_test.go index 436af4c2e..0a531baf4 100644 --- a/tasks/messages/message_test.go +++ b/tasks/messages/message_test.go @@ -174,7 +174,7 @@ func TestParseMessageParams(t *testing.T) { }, }), wantMethod: "PublishStorageDeals", - wantEncoded: `{"Deals":[{"Proposal":{"PieceCID":{"/":"baga6ea4seaqgqzxo27ongakwwef5x3cihl6fgritvgeq5akvjqij6lpgofsogiq"},"PieceSize":1310720,"VerifiedDeal":false,"Client":"f1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i","Provider":"f08178",` + + wantEncoded: `{"Deals":[{"Proposal":{"PieceCID":{"/":"baga6ea4seaqgqzxo27ongakwwef5x3cihl6fgritvgeq5akvjqij6lpgofsogiq"},"PieceSize":1310720,"VerifiedDeal":false,"Client":"f1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i","Client":"f08178",` + `"Label":"\\ufffdepcids\\ufffd\\ufffd*X'\\u0000\\u0001U\\ufffd\\ufffd\\u0002 \\u0011;\\u0012\\ufffd\\ufffd0\\ufffd3\\ufffdMA\\ufffd\\ufffd}b\\ufffd\\rf\\ufffdmX\\u001b\u003e\\ufffd\\ufffdm\\ufffd۬\\ufffd\\ufffd\\ufffd",` + `"StartEpoch":475750,"EndEpoch":750173,"StoragePricePerEpoch":{"Int":61035},"ProviderCollateral":{"Int":0},"ClientCollateral":{"Int":0}},"ClientSignature":{"Type":1,"Data":"9a8sdvutVlu0fizD0JmqZjKJaQLj3W3ZtJ2yTReIry8kZ8cDa33V3Pe0sdZzSjz9mRdM/KPm1jL/PZhqpDeYNwE="}}]}`, wantErr: false, diff --git a/testutil/lens.go b/testutil/lens.go index b17428e94..cbf8d87a2 100644 --- a/testutil/lens.go +++ b/testutil/lens.go @@ -25,6 +25,11 @@ type APIWrapper struct { ctx context.Context } +func (aw *APIWrapper) GetMessageExecutionsForTipSetV2(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecutionV2, error) { + //TODO implement me + panic("implement me") +} + func (aw *APIWrapper) BurnFundsFn(ctx context.Context, ts *types.TipSet) (lens.ShouldBurnFn, error) { //TODO implement me panic("implement me")