diff --git a/.bingo/.gitignore b/.bingo/.gitignore index 9efccf6..4f2055b 100755 --- a/.bingo/.gitignore +++ b/.bingo/.gitignore @@ -5,7 +5,6 @@ # But not these files: !.gitignore !*.mod -!*.sum !README.md !Variables.mk !variables.env diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk index d535073..410a7ac 100644 --- a/.bingo/Variables.mk +++ b/.bingo/Variables.mk @@ -20,35 +20,35 @@ BUF := $(GOBIN)/buf-v0.20.5 $(BUF): .bingo/buf.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/buf-v0.20.5" - @cd .bingo && $(GO) build -modfile=buf.mod -o=$(GOBIN)/buf-v0.20.5 "github.com/bufbuild/buf/cmd/buf" + @cd .bingo && $(GO) build -mod=mod -modfile=buf.mod -o=$(GOBIN)/buf-v0.20.5 "github.com/bufbuild/buf/cmd/buf" GOMPLATE := $(GOBIN)/gomplate-v3.8.0 $(GOMPLATE): .bingo/gomplate.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/gomplate-v3.8.0" - @cd .bingo && $(GO) build -modfile=gomplate.mod -o=$(GOBIN)/gomplate-v3.8.0 "github.com/hairyhenderson/gomplate/v3/cmd/gomplate" + @cd .bingo && $(GO) build -mod=mod -modfile=gomplate.mod -o=$(GOBIN)/gomplate-v3.8.0 "github.com/hairyhenderson/gomplate/v3/cmd/gomplate" GOVVV := $(GOBIN)/govvv-v0.3.0 $(GOVVV): .bingo/govvv.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/govvv-v0.3.0" - @cd .bingo && $(GO) build -modfile=govvv.mod -o=$(GOBIN)/govvv-v0.3.0 "github.com/ahmetb/govvv" + @cd .bingo && $(GO) build -mod=mod -modfile=govvv.mod -o=$(GOBIN)/govvv-v0.3.0 "github.com/ahmetb/govvv" GOX := $(GOBIN)/gox-v1.0.1 $(GOX): .bingo/gox.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/gox-v1.0.1" - @cd .bingo && $(GO) build -modfile=gox.mod -o=$(GOBIN)/gox-v1.0.1 "github.com/mitchellh/gox" + @cd .bingo && $(GO) build -mod=mod -modfile=gox.mod -o=$(GOBIN)/gox-v1.0.1 "github.com/mitchellh/gox" PROTOC_GEN_BUF_CHECK_BREAKING := $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 $(PROTOC_GEN_BUF_CHECK_BREAKING): .bingo/protoc-gen-buf-check-breaking.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5" - @cd .bingo && $(GO) build -modfile=protoc-gen-buf-check-breaking.mod -o=$(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-breaking" + @cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-breaking.mod -o=$(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-breaking" PROTOC_GEN_BUF_CHECK_LINT := $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 $(PROTOC_GEN_BUF_CHECK_LINT): .bingo/protoc-gen-buf-check-lint.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5" - @cd .bingo && $(GO) build -modfile=protoc-gen-buf-check-lint.mod -o=$(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-lint" + @cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-lint.mod -o=$(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-lint" diff --git a/.bingo/buf.mod b/.bingo/buf.mod index 8b53089..7e2a9f8 100644 --- a/.bingo/buf.mod +++ b/.bingo/buf.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.16 +go 1.14 require github.com/bufbuild/buf v0.20.5 // cmd/buf diff --git a/.bingo/go.mod b/.bingo/go.mod index ac35c85..79cf705 100755 --- a/.bingo/go.mod +++ b/.bingo/go.mod @@ -1,3 +1,3 @@ module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. -go 1.16 +go 1.14 diff --git a/.bingo/gomplate.mod b/.bingo/gomplate.mod index 642aae4..c458a0a 100644 --- a/.bingo/gomplate.mod +++ b/.bingo/gomplate.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.16 +go 1.14 require github.com/hairyhenderson/gomplate/v3 v3.8.0 // cmd/gomplate diff --git a/.bingo/govvv.mod b/.bingo/govvv.mod index acde2ee..b5112d9 100644 --- a/.bingo/govvv.mod +++ b/.bingo/govvv.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.16 +go 1.14 require github.com/ahmetb/govvv v0.3.0 diff --git a/.bingo/govvv.sum b/.bingo/govvv.sum deleted file mode 100644 index ad6764f..0000000 --- a/.bingo/govvv.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0= -github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s= diff --git a/.bingo/gox.mod b/.bingo/gox.mod index ca28943..2afa42a 100644 --- a/.bingo/gox.mod +++ b/.bingo/gox.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.16 +go 1.14 require github.com/mitchellh/gox v1.0.1 diff --git a/.bingo/protoc-gen-buf-check-breaking.mod b/.bingo/protoc-gen-buf-check-breaking.mod index c7de7d9..9fe534c 100644 --- a/.bingo/protoc-gen-buf-check-breaking.mod +++ b/.bingo/protoc-gen-buf-check-breaking.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.16 +go 1.14 require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-breaking diff --git a/.bingo/protoc-gen-buf-check-lint.mod b/.bingo/protoc-gen-buf-check-lint.mod index 3350e78..238d059 100644 --- a/.bingo/protoc-gen-buf-check-lint.mod +++ b/.bingo/protoc-gen-buf-check-lint.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.16 +go 1.14 require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-lint diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06a1005..0791658 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,11 +24,8 @@ jobs: buck: name: Buck CLI runs-on: ubuntu-latest + container: golang:1.16.0-buster steps: - - name: setup - uses: actions/setup-go@v1 - with: - go-version: 1.16 - name: checkout uses: actions/checkout@v1 - name: build @@ -36,11 +33,8 @@ jobs: buckd: name: Buck Daemon runs-on: ubuntu-latest + container: golang:1.16.0-buster steps: - - name: setup - uses: actions/setup-go@v1 - with: - go-version: 1.16 - name: checkout uses: actions/checkout@v1 - name: build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed66fed..81a952a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,11 +6,8 @@ jobs: release-platform-builds: name: Release Builds runs-on: ubuntu-latest + container: golang:1.16.0-buster steps: - - name: Set up Go - uses: actions/setup-go@v1 - with: - go-version: 1.16 - name: Check out code uses: actions/checkout@v1 - name: Cache dependencies diff --git a/README.md b/README.md index c3a73ac..4ef520b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Join us on our [public Slack channel](https://slack.textile.io/) for news, discussions, and status updates. [Check out our blog](https://medium.com/textileio) for the latest posts and announcements. +### WARNING: This repo is pointing to a [feature branch](https://github.com/textileio/go-threads/pull/498) of `go-threads` that handles identities as Decentralized Identifiers (DIDs) and should be considered alpha. The [textile repo](https://github.com/textileio/textile) contains the non-DID-based bucket implementation currently compatible with [Textile's public hub](https://cloud.hub.textile.io/#/access). DID-based buckets will be integrated into the hub mid 2021. In the meantime, you can still use this repo for standalone bucket peers. + ## Table of Contents - [Security](#security) diff --git a/access.go b/access.go index c1bf925..a16eccf 100644 --- a/access.go +++ b/access.go @@ -14,28 +14,48 @@ import ( core "github.com/textileio/go-threads/core/thread" ) +// PushPathAccessRoles pushes new access roles to bucket paths. +// Access roles are keyed by did.DID. +// Roles are inherited by path children. func (b *Buckets) PushPathAccessRoles( ctx context.Context, thread core.ID, - key, pth string, - roles map[did.DID]collection.Role, + key string, identity did.Token, + root path.Resolved, + pth string, + roles map[did.DID]collection.Role, ) (int64, *Bucket, error) { - lk := b.locks.Get(lock(key)) - lk.Acquire() - defer lk.Release() + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + return 0, nil, err + } + defer txn.Close() + return txn.PushPathAccessRoles(ctx, root, pth, roles) +} +// PushPathAccessRoles is Txn based PushPathInfo. +func (t *Txn) PushPathAccessRoles( + ctx context.Context, + root path.Resolved, + pth string, + roles map[did.DID]collection.Role, +) (int64, *Bucket, error) { pth, err := parsePath(pth) if err != nil { return 0, nil, err } - instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity) + instance, bpth, err := t.b.getBucketAndPath(ctx, t.thread, t.key, t.identity, pth) if err != nil { return 0, nil, err } + if root != nil && root.String() != instance.Path { + return 0, nil, ErrNonFastForward + } + linkKey := instance.GetLinkEncryptionKey() - pathNode, err := dag.GetNodeAtPath(ctx, b.ipfs, bpth, linkKey) + pathNode, err := dag.GetNodeAtPath(ctx, t.b.ipfs, bpth, linkKey) if err != nil { return 0, nil, err } @@ -86,7 +106,7 @@ func (b *Buckets) PushPathAccessRoles( return 0, nil, err } } - if err := b.c.Verify(ctx, thread, instance, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { return 0, nil, err } @@ -97,7 +117,7 @@ func (b *Buckets) PushPathAccessRoles( } nmap, err := dag.EncryptDag( ctx, - b.ipfs, + t.b.ipfs, pathNode, pth, linkKey, @@ -117,41 +137,46 @@ func (b *Buckets) PushPathAccessRoles( } pn := nmap[pathNode.Cid()].Node var dirPath path.Resolved - ctx, dirPath, err = dag.InsertNodeAtPath(ctx, b.ipfs, pn, path.Join(path.New(instance.Path), pth), linkKey) + ctx, dirPath, err = dag.InsertNodeAtPath(ctx, t.b.ipfs, pn, path.Join(path.New(instance.Path), pth), linkKey) if err != nil { return 0, nil, err } - ctx, err = dag.AddAndPinNodes(ctx, b.ipfs, nodes) + ctx, err = dag.AddAndPinNodes(ctx, t.b.ipfs, nodes) if err != nil { return 0, nil, err } instance.Path = dirPath.String() } - if err := b.c.Save(ctx, thread, instance, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Save(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { return 0, nil, err } } - log.Debugf("pushed access roles for %s in %s", pth, key) - return dag.GetPinnedBytes(ctx), instanceToBucket(thread, instance), nil + log.Debugf("pushed access roles for %s in %s", pth, t.key) + return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil } +// PullPathAccessRoles pulls access roles for a bucket path. func (b *Buckets) PullPathAccessRoles( ctx context.Context, thread core.ID, - key, pth string, + key string, identity did.Token, + pth string, ) (map[did.DID]collection.Role, error) { + if err := thread.Validate(); err != nil { + return nil, fmt.Errorf("invalid thread id: %v", err) + } pth, err := parsePath(pth) if err != nil { return nil, err } - instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity) + instance, bpth, err := b.getBucketAndPath(ctx, thread, key, identity, pth) if err != nil { return nil, err } - if _, err := dag.GetNodeAtPath(ctx, b.ipfs, bpth, instance.GetLinkEncryptionKey()); err != nil { + if _, err = dag.GetNodeAtPath(ctx, b.ipfs, bpth, instance.GetLinkEncryptionKey()); err != nil { return nil, fmt.Errorf("could not resolve path: %s", pth) } md, _, ok := instance.GetMetadataForPath(pth, false) @@ -162,3 +187,55 @@ func (b *Buckets) PullPathAccessRoles( log.Debugf("pulled access roles for %s in %s", pth, key) return md.Roles, nil } + +// IsReadablePath returns whether or not a path is readable by an identity. +func (b *Buckets) IsReadablePath( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + pth string, +) (bool, error) { + if err := thread.Validate(); err != nil { + return false, fmt.Errorf("invalid thread id: %v", err) + } + pth, err := parsePath(pth) + if err != nil { + return false, err + } + instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) + if err != nil { + return false, err + } + _, doc, err := core.Validate(identity, nil) + if err != nil { + return false, err + } + return instance.IsReadablePath(pth, doc.ID), nil +} + +// IsWritablePath returns whether or not a path is writable by an identity. +func (b *Buckets) IsWritablePath( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + pth string, +) (bool, error) { + if err := thread.Validate(); err != nil { + return false, fmt.Errorf("invalid thread id: %v", err) + } + pth, err := parsePath(pth) + if err != nil { + return false, err + } + instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) + if err != nil { + return false, err + } + _, doc, err := core.Validate(identity, nil) + if err != nil { + return false, err + } + return instance.IsWritablePath(pth, doc.ID), nil +} diff --git a/api/apitest/apitest.go b/api/apitest/apitest.go index 58467cb..6cfc464 100644 --- a/api/apitest/apitest.go +++ b/api/apitest/apitest.go @@ -19,7 +19,6 @@ import ( "github.com/textileio/go-buckets" "github.com/textileio/go-buckets/api/common" "github.com/textileio/go-buckets/ipns" - "github.com/textileio/go-buckets/util" dbc "github.com/textileio/go-threads/api/client" "github.com/textileio/go-threads/core/did" tdb "github.com/textileio/go-threads/db" @@ -30,9 +29,9 @@ import ( func NewService(t *testing.T) (listenAddr string, host did.DID) { err := tutil.SetLogLevels(map[string]logging.LogLevel{ "buckets": logging.LevelDebug, - "buckets-api": logging.LevelDebug, - "buckets-ipns": logging.LevelDebug, - "buckets-dns": logging.LevelDebug, + "buckets/api": logging.LevelDebug, + "buckets/ipns": logging.LevelDebug, + "buckets/dns": logging.LevelDebug, }) require.NoError(t, err) @@ -40,7 +39,7 @@ func NewService(t *testing.T) (listenAddr string, host did.DID) { net, err := nc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...) require.NoError(t, err) - // @todo: Fix me + // @todo: Use service description to build client doc, err := net.GetServices(context.Background()) require.NoError(t, err) @@ -48,7 +47,8 @@ func NewService(t *testing.T) (listenAddr string, host did.DID) { require.NoError(t, err) ipfs, err := httpapi.NewApi(GetIPFSApiMultiAddr()) require.NoError(t, err) - ipnsm, err := ipns.NewManager(tdb.NewTxMapDatastore(), ipfs) + ipnsms := tdb.NewTxMapDatastore() + ipnsm, err := ipns.NewManager(ipnsms, ipfs) require.NoError(t, err) lib, err := buckets.NewBuckets(net, db, ipfs, ipnsm, nil) require.NoError(t, err) @@ -64,6 +64,8 @@ func NewService(t *testing.T) (listenAddr string, host did.DID) { server.Stop() require.NoError(t, lib.Close()) require.NoError(t, ipnsm.Close()) + require.NoError(t, ipnsms.Close()) + require.NoError(t, db.Close()) require.NoError(t, net.Close()) }) @@ -83,9 +85,9 @@ func GetThreadsApiAddr() string { func GetIPFSApiMultiAddr() ma.Multiaddr { env := os.Getenv("IPFS_API_MULTIADDR") if env != "" { - return util.MustParseAddr(env) + return tutil.MustParseAddr(env) } - return util.MustParseAddr("/ip4/127.0.0.1/tcp/5012") + return tutil.MustParseAddr("/ip4/127.0.0.1/tcp/5012") } // StartServices starts an ipfs and threads node for tests. diff --git a/api/apitest/docker-compose.yml b/api/apitest/docker-compose.yml index 8f1651b..d96237b 100644 --- a/api/apitest/docker-compose.yml +++ b/api/apitest/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: threads: - image: textile/go-threads:534a6d0 + image: textile/go-threads:f604018-m1 environment: - THREADS_APIADDR=/ip4/0.0.0.0/tcp/5000 - THREADS_APIPROXYADDR=/ip4/0.0.0.0/tcp/5050 @@ -11,7 +11,7 @@ services: - "127.0.0.1:4002:5000" - "127.0.0.1:4052:5050" ipfs: - image: ipfs/go-ipfs:v0.8.0 + image: textile/go-ipfs:v0.8.0-m1 environment: - IPFS_PROFILE=test ports: diff --git a/api/cast/cast.go b/api/cast/cast.go index 08da3b6..223b3d9 100644 --- a/api/cast/cast.go +++ b/api/cast/cast.go @@ -1,6 +1,8 @@ package cast import ( + "encoding/json" + "github.com/textileio/go-buckets" pb "github.com/textileio/go-buckets/api/pb/buckets" "github.com/textileio/go-buckets/collection" @@ -57,18 +59,28 @@ func BucketFromPb(bucket *pb.Bucket) (buckets.Bucket, error) { } func MetadataToPb(md collection.Metadata) *pb.Metadata { + var info []byte + if md.Info != nil { + info, _ = json.Marshal(md.Info) + } return &pb.Metadata{ Key: md.Key, Roles: RolesToPb(md.Roles), UpdatedAt: md.UpdatedAt, + Info: info, } } func MetadataFromPb(md *pb.Metadata) collection.Metadata { + var info map[string]interface{} + if md.Info != nil { + _ = json.Unmarshal(md.Info, &info) + } return collection.Metadata{ Key: md.Key, Roles: RolesFromPb(md.Roles), UpdatedAt: md.UpdatedAt, + Info: info, } } @@ -127,11 +139,24 @@ func RolesFromPb(roles map[string]pb.PathAccessRole) map[did.DID]collection.Role return croles } +func InfoToPb(info map[string]interface{}) ([]byte, error) { + return json.Marshal(info) +} + +func InfoFromPb(info []byte) (map[string]interface{}, error) { + var pinfo map[string]interface{} + if err := json.Unmarshal(info, &pinfo); err != nil { + return nil, err + } + return pinfo, nil +} + func LinksToPb(links buckets.Links) *pb.Links { return &pb.Links{ Url: links.URL, Www: links.WWW, Ipns: links.IPNS, + Bps: links.BPS, } } @@ -140,5 +165,6 @@ func LinksFromPb(links *pb.Links) buckets.Links { URL: links.Url, WWW: links.Www, IPNS: links.Ipns, + BPS: links.Bps, } } diff --git a/api/client/client.go b/api/client/client.go index 81d900b..3239b50 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -18,7 +18,7 @@ import ( "github.com/textileio/go-buckets/api/cast" pb "github.com/textileio/go-buckets/api/pb/buckets" "github.com/textileio/go-buckets/collection" - "github.com/textileio/go-buckets/util" + "github.com/textileio/go-buckets/dag" "github.com/textileio/go-threads/core/did" core "github.com/textileio/go-threads/core/thread" "google.golang.org/grpc" @@ -41,25 +41,6 @@ type Client struct { target did.DID } -// /ip4//tcp//p2p/ -// /dnsaddr//p2p/ -//func getGRPCTarget(addr maddr.Multiaddr) (string, error) { -// p2p, err := addr.ValueForProtocol(maddr.P_P2P) -// if err != nil { -// return "", fmt.Errorf("p2p address component is missing from %s", addr) -// } -// ip, err := addr.ValueForProtocol(maddr.P_IP4) -// if err == nil { -// ip, err := addr.ValueForProtocol(maddr.P_IP4) -// return "", fmt.Errorf("p2p address component is missing from %s", addr) -// } else { -// dnsaddr, err := addr.ValueForProtocol(maddr.P_DNS) -// if err != nil { -// -// } -// } -//} - // NewClient starts the client. func NewClient(addr string, opts ...grpc.DialOption) (*Client, error) { conn, err := grpc.Dial(addr, opts...) @@ -69,7 +50,7 @@ func NewClient(addr string, opts ...grpc.DialOption) (*Client, error) { return &Client{ c: pb.NewAPIServiceClient(conn), conn: conn, - target: "did:key:foo", // @todo: Fix me + target: "did:key:foo", // @todo: Get target from thread services }, nil } @@ -93,10 +74,7 @@ func (c *Client) NewTokenContext( // Create initializes a new bucket. // The bucket name is only meant to help identify a bucket in a UI and is not unique. -func (c *Client) Create( - ctx context.Context, - opts ...buckets.CreateOption, -) (*pb.CreateResponse, error) { +func (c *Client) Create(ctx context.Context, opts ...buckets.CreateOption) (*pb.CreateResponse, error) { args := &buckets.CreateOptions{} for _, opt := range opts { opt(args) @@ -438,7 +416,7 @@ func (c *Client) PushPaths( q.outCh <- PushPathsResult{err: err} return } - root, err := util.NewResolvedPath(rep.Bucket.Path) + root, err := dag.NewResolvedPath(rep.Bucket.Path) if err != nil { q.outCh <- PushPathsResult{err: err} return @@ -455,10 +433,11 @@ func (c *Client) PushPaths( sendChunk := func(c *pb.PushPathsRequest_Chunk) bool { q.lk.Lock() - defer q.lk.Unlock() if q.closed { + q.lk.Unlock() return false } + q.lk.Unlock() if err := stream.Send(&pb.PushPathsRequest{ Payload: &pb.PushPathsRequest_Chunk_{ @@ -471,9 +450,12 @@ func (c *Client) PushPaths( return false } atomic.AddInt64(&q.complete, int64(len(c.Data))) - if args.Progress != nil { - args.Progress <- q.complete + + q.lk.Lock() + if !q.closed && args.Progress != nil { + args.Progress <- atomic.LoadInt64(&q.complete) } + q.lk.Unlock() return true } @@ -592,20 +574,46 @@ func (c *Client) SetPath( thread core.ID, key, pth string, remoteCid cid.Cid, + opts ...buckets.Option, ) (*pb.SetPathResponse, error) { + args := &buckets.Options{} + for _, opt := range opts { + opt(args) + } + + var xr string + if args.Root != nil { + xr = args.Root.String() + } return c.c.SetPath(ctx, &pb.SetPathRequest{ Thread: thread.String(), Key: key, + Root: xr, Path: filepath.ToSlash(pth), Cid: remoteCid.String(), }) } // MovePath moves a particular path to another path in the existing IPFS UnixFS DAG. -func (c *Client) MovePath(ctx context.Context, thread core.ID, key, pth string, dest string) error { +func (c *Client) MovePath( + ctx context.Context, + thread core.ID, + key, pth string, dest string, + opts ...buckets.Option, +) error { + args := &buckets.Options{} + for _, opt := range opts { + opt(args) + } + + var xr string + if args.Root != nil { + xr = args.Root.String() + } _, err := c.c.MovePath(ctx, &pb.MovePathRequest{ Thread: thread.String(), Key: key, + Root: xr, FromPath: filepath.ToSlash(pth), ToPath: filepath.ToSlash(dest), }) @@ -624,6 +632,7 @@ func (c *Client) RemovePath( for _, opt := range opts { opt(args) } + var xr string if args.Root != nil { xr = args.Root.String() @@ -631,13 +640,13 @@ func (c *Client) RemovePath( res, err := c.c.RemovePath(ctx, &pb.RemovePathRequest{ Thread: thread.String(), Key: key, - Path: filepath.ToSlash(pth), Root: xr, + Path: filepath.ToSlash(pth), }) if err != nil { return nil, err } - return util.NewResolvedPath(res.Bucket.Path) + return dag.NewResolvedPath(res.Bucket.Path) } // PushPathAccessRoles updates path access roles by merging the pushed roles with existing roles. @@ -649,10 +658,21 @@ func (c *Client) PushPathAccessRoles( thread core.ID, key, pth string, roles map[did.DID]collection.Role, + opts ...buckets.Option, ) error { + args := &buckets.Options{} + for _, opt := range opts { + opt(args) + } + + var xr string + if args.Root != nil { + xr = args.Root.String() + } _, err := c.c.PushPathAccessRoles(ctx, &pb.PushPathAccessRolesRequest{ Thread: thread.String(), Key: key, + Root: xr, Path: filepath.ToSlash(pth), Roles: cast.RolesToPb(roles), }) @@ -675,3 +695,52 @@ func (c *Client) PullPathAccessRoles( } return cast.RolesFromPb(res.Roles), nil } + +// PushPathInfo updates path info by merging the pushed info with existing info. +func (c *Client) PushPathInfo( + ctx context.Context, + thread core.ID, + key, pth string, + info map[string]interface{}, + opts ...buckets.Option, +) error { + args := &buckets.Options{} + for _, opt := range opts { + opt(args) + } + + var xr string + if args.Root != nil { + xr = args.Root.String() + } + + data, err := cast.InfoToPb(info) + if err != nil { + return err + } + _, err = c.c.PushPathInfo(ctx, &pb.PushPathInfoRequest{ + Thread: thread.String(), + Key: key, + Root: xr, + Path: filepath.ToSlash(pth), + Info: data, + }) + return err +} + +// PullPathInfo returns info for a path. +func (c *Client) PullPathInfo( + ctx context.Context, + thread core.ID, + key, pth string, +) (map[string]interface{}, error) { + res, err := c.c.PullPathInfo(ctx, &pb.PullPathInfoRequest{ + Thread: thread.String(), + Key: key, + Path: filepath.ToSlash(pth), + }) + if err != nil { + return nil, err + } + return cast.InfoFromPb(res.Info) +} diff --git a/api/client/client_test.go b/api/client/client_test.go index d2424c0..770b0cc 100644 --- a/api/client/client_test.go +++ b/api/client/client_test.go @@ -25,7 +25,7 @@ import ( "github.com/textileio/go-buckets/api/client" "github.com/textileio/go-buckets/api/common" "github.com/textileio/go-buckets/collection" - "github.com/textileio/go-buckets/util" + "github.com/textileio/go-buckets/dag" "github.com/textileio/go-threads/core/did" "github.com/textileio/go-threads/core/thread" ) @@ -385,7 +385,7 @@ func pushPaths(t *testing.T, ctx context.Context, c *client.Client, private bool assert.Len(t, rep3.Item.Items, 3) // Concurrent writes should result in one being rejected due to the fast-forward-only rule - root, err := util.NewResolvedPath(rep3.Bucket.Path) + root, err := dag.NewResolvedPath(rep3.Bucket.Path) require.NoError(t, err) var err1, err2 error var wg sync.WaitGroup @@ -1091,6 +1091,8 @@ func checkAccess(t *testing.T, ctx context.Context, c *client.Client, check acce defer tmp.Close() _, err = io.CopyN(tmp, rand.Reader, 1024) require.NoError(t, err) + _, err = tmp.Seek(0, 0) + require.NoError(t, err) q, err := c.PushPaths(ctx, check.Thread, check.Key) require.NoError(t, err) err = q.AddReader(check.Path, tmp, 0) diff --git a/api/common/common.go b/api/common/common.go index 2be313b..c8c7160 100644 --- a/api/common/common.go +++ b/api/common/common.go @@ -17,7 +17,7 @@ import ( "google.golang.org/grpc/credentials" ) -var log = logging.Logger("buckets-api") +var log = logging.Logger("buckets/api") func GetClientRPCOpts(target string) (opts []grpc.DialOption) { creds := did.RPCCredentials{} diff --git a/api/pb/buckets/buckets.pb.go b/api/pb/buckets/buckets.pb.go index 6b1ae8f..7a6f402 100644 --- a/api/pb/buckets/buckets.pb.go +++ b/api/pb/buckets/buckets.pb.go @@ -88,7 +88,8 @@ type Metadata struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` Roles map[string]PathAccessRole `protobuf:"bytes,2,rep,name=roles,proto3" json:"roles,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=api.pb.buckets.PathAccessRole"` - UpdatedAt int64 `protobuf:"varint,3,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` + Info []byte `protobuf:"bytes,3,opt,name=info,proto3" json:"info,omitempty"` + UpdatedAt int64 `protobuf:"varint,4,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` } func (x *Metadata) Reset() { @@ -137,6 +138,13 @@ func (x *Metadata) GetRoles() map[string]PathAccessRole { return nil } +func (x *Metadata) GetInfo() []byte { + if x != nil { + return x.Info + } + return nil +} + func (x *Metadata) GetUpdatedAt() int64 { if x != nil { return x.UpdatedAt @@ -271,6 +279,7 @@ type Links struct { Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` Www string `protobuf:"bytes,2,opt,name=www,proto3" json:"www,omitempty"` Ipns string `protobuf:"bytes,3,opt,name=ipns,proto3" json:"ipns,omitempty"` + Bps string `protobuf:"bytes,4,opt,name=bps,proto3" json:"bps,omitempty"` } func (x *Links) Reset() { @@ -326,6 +335,13 @@ func (x *Links) GetIpns() string { return "" } +func (x *Links) GetBps() string { + if x != nil { + return x.Bps + } + return "" +} + type Seed struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1632,8 +1648,9 @@ type SetPathRequest struct { Thread string `protobuf:"bytes,1,opt,name=thread,proto3" json:"thread,omitempty"` Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` - Cid string `protobuf:"bytes,4,opt,name=cid,proto3" json:"cid,omitempty"` + Root string `protobuf:"bytes,3,opt,name=root,proto3" json:"root,omitempty"` + Path string `protobuf:"bytes,4,opt,name=path,proto3" json:"path,omitempty"` + Cid string `protobuf:"bytes,5,opt,name=cid,proto3" json:"cid,omitempty"` } func (x *SetPathRequest) Reset() { @@ -1682,6 +1699,13 @@ func (x *SetPathRequest) GetKey() string { return "" } +func (x *SetPathRequest) GetRoot() string { + if x != nil { + return x.Root + } + return "" +} + func (x *SetPathRequest) GetPath() string { if x != nil { return x.Path @@ -1758,8 +1782,9 @@ type MovePathRequest struct { Thread string `protobuf:"bytes,1,opt,name=thread,proto3" json:"thread,omitempty"` Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - FromPath string `protobuf:"bytes,3,opt,name=from_path,json=fromPath,proto3" json:"from_path,omitempty"` - ToPath string `protobuf:"bytes,4,opt,name=to_path,json=toPath,proto3" json:"to_path,omitempty"` + Root string `protobuf:"bytes,3,opt,name=root,proto3" json:"root,omitempty"` + FromPath string `protobuf:"bytes,4,opt,name=from_path,json=fromPath,proto3" json:"from_path,omitempty"` + ToPath string `protobuf:"bytes,5,opt,name=to_path,json=toPath,proto3" json:"to_path,omitempty"` } func (x *MovePathRequest) Reset() { @@ -1808,6 +1833,13 @@ func (x *MovePathRequest) GetKey() string { return "" } +func (x *MovePathRequest) GetRoot() string { + if x != nil { + return x.Root + } + return "" +} + func (x *MovePathRequest) GetFromPath() string { if x != nil { return x.FromPath @@ -1884,8 +1916,8 @@ type RemovePathRequest struct { Thread string `protobuf:"bytes,1,opt,name=thread,proto3" json:"thread,omitempty"` Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` - Root string `protobuf:"bytes,4,opt,name=root,proto3" json:"root,omitempty"` + Root string `protobuf:"bytes,3,opt,name=root,proto3" json:"root,omitempty"` + Path string `protobuf:"bytes,4,opt,name=path,proto3" json:"path,omitempty"` } func (x *RemovePathRequest) Reset() { @@ -1934,16 +1966,16 @@ func (x *RemovePathRequest) GetKey() string { return "" } -func (x *RemovePathRequest) GetPath() string { +func (x *RemovePathRequest) GetRoot() string { if x != nil { - return x.Path + return x.Root } return "" } -func (x *RemovePathRequest) GetRoot() string { +func (x *RemovePathRequest) GetPath() string { if x != nil { - return x.Root + return x.Path } return "" } @@ -2011,7 +2043,8 @@ type PushPathAccessRolesRequest struct { Thread string `protobuf:"bytes,1,opt,name=thread,proto3" json:"thread,omitempty"` Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` - Roles map[string]PathAccessRole `protobuf:"bytes,4,rep,name=roles,proto3" json:"roles,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=api.pb.buckets.PathAccessRole"` + Root string `protobuf:"bytes,4,opt,name=root,proto3" json:"root,omitempty"` + Roles map[string]PathAccessRole `protobuf:"bytes,5,rep,name=roles,proto3" json:"roles,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=api.pb.buckets.PathAccessRole"` } func (x *PushPathAccessRolesRequest) Reset() { @@ -2067,6 +2100,13 @@ func (x *PushPathAccessRolesRequest) GetPath() string { return "" } +func (x *PushPathAccessRolesRequest) GetRoot() string { + if x != nil { + return x.Root + } + return "" +} + func (x *PushPathAccessRolesRequest) GetRoles() map[string]PathAccessRole { if x != nil { return x.Roles @@ -2239,6 +2279,242 @@ func (x *PullPathAccessRolesResponse) GetRoles() map[string]PathAccessRole { return nil } +type PushPathInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Thread string `protobuf:"bytes,1,opt,name=thread,proto3" json:"thread,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` + Root string `protobuf:"bytes,4,opt,name=root,proto3" json:"root,omitempty"` + Info []byte `protobuf:"bytes,5,opt,name=info,proto3" json:"info,omitempty"` +} + +func (x *PushPathInfoRequest) Reset() { + *x = PushPathInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PushPathInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PushPathInfoRequest) ProtoMessage() {} + +func (x *PushPathInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[35] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PushPathInfoRequest.ProtoReflect.Descriptor instead. +func (*PushPathInfoRequest) Descriptor() ([]byte, []int) { + return file_api_pb_buckets_buckets_proto_rawDescGZIP(), []int{35} +} + +func (x *PushPathInfoRequest) GetThread() string { + if x != nil { + return x.Thread + } + return "" +} + +func (x *PushPathInfoRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *PushPathInfoRequest) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *PushPathInfoRequest) GetRoot() string { + if x != nil { + return x.Root + } + return "" +} + +func (x *PushPathInfoRequest) GetInfo() []byte { + if x != nil { + return x.Info + } + return nil +} + +type PushPathInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Bucket *Bucket `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` +} + +func (x *PushPathInfoResponse) Reset() { + *x = PushPathInfoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PushPathInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PushPathInfoResponse) ProtoMessage() {} + +func (x *PushPathInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[36] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PushPathInfoResponse.ProtoReflect.Descriptor instead. +func (*PushPathInfoResponse) Descriptor() ([]byte, []int) { + return file_api_pb_buckets_buckets_proto_rawDescGZIP(), []int{36} +} + +func (x *PushPathInfoResponse) GetBucket() *Bucket { + if x != nil { + return x.Bucket + } + return nil +} + +type PullPathInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Thread string `protobuf:"bytes,1,opt,name=thread,proto3" json:"thread,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` +} + +func (x *PullPathInfoRequest) Reset() { + *x = PullPathInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PullPathInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PullPathInfoRequest) ProtoMessage() {} + +func (x *PullPathInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[37] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PullPathInfoRequest.ProtoReflect.Descriptor instead. +func (*PullPathInfoRequest) Descriptor() ([]byte, []int) { + return file_api_pb_buckets_buckets_proto_rawDescGZIP(), []int{37} +} + +func (x *PullPathInfoRequest) GetThread() string { + if x != nil { + return x.Thread + } + return "" +} + +func (x *PullPathInfoRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *PullPathInfoRequest) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +type PullPathInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Info []byte `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"` +} + +func (x *PullPathInfoResponse) Reset() { + *x = PullPathInfoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PullPathInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PullPathInfoResponse) ProtoMessage() {} + +func (x *PullPathInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_pb_buckets_buckets_proto_msgTypes[38] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PullPathInfoResponse.ProtoReflect.Descriptor instead. +func (*PullPathInfoResponse) Descriptor() ([]byte, []int) { + return file_api_pb_buckets_buckets_proto_rawDescGZIP(), []int{38} +} + +func (x *PullPathInfoResponse) GetInfo() []byte { + if x != nil { + return x.Info + } + return nil +} + type PushPathsRequest_Header struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2252,7 +2528,7 @@ type PushPathsRequest_Header struct { func (x *PushPathsRequest_Header) Reset() { *x = PushPathsRequest_Header{} if protoimpl.UnsafeEnabled { - mi := &file_api_pb_buckets_buckets_proto_msgTypes[37] + mi := &file_api_pb_buckets_buckets_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2265,7 +2541,7 @@ func (x *PushPathsRequest_Header) String() string { func (*PushPathsRequest_Header) ProtoMessage() {} func (x *PushPathsRequest_Header) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_buckets_buckets_proto_msgTypes[37] + mi := &file_api_pb_buckets_buckets_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2314,7 +2590,7 @@ type PushPathsRequest_Chunk struct { func (x *PushPathsRequest_Chunk) Reset() { *x = PushPathsRequest_Chunk{} if protoimpl.UnsafeEnabled { - mi := &file_api_pb_buckets_buckets_proto_msgTypes[38] + mi := &file_api_pb_buckets_buckets_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2327,7 +2603,7 @@ func (x *PushPathsRequest_Chunk) String() string { func (*PushPathsRequest_Chunk) ProtoMessage() {} func (x *PushPathsRequest_Chunk) ProtoReflect() protoreflect.Message { - mi := &file_api_pb_buckets_buckets_proto_msgTypes[38] + mi := &file_api_pb_buckets_buckets_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2362,351 +2638,389 @@ var File_api_pb_buckets_buckets_proto protoreflect.FileDescriptor var file_api_pb_buckets_buckets_proto_rawDesc = []byte{ 0x0a, 0x1c, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x2f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, - 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0xd0, + 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0xe4, 0x01, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x58, 0x0a, 0x0a, 0x52, 0x6f, 0x6c, 0x65, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0xfc, 0x02, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, - 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x6c, 0x69, - 0x6e, 0x6b, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, 0x69, - 0x6e, 0x6b, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x40, 0x0a, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x55, 0x0a, 0x0d, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x3f, 0x0a, 0x05, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x77, - 0x77, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x77, 0x77, 0x77, 0x12, 0x12, 0x0a, - 0x04, 0x69, 0x70, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x70, 0x6e, - 0x73, 0x22, 0x2c, 0x0a, 0x04, 0x53, 0x65, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, - 0x67, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x79, 0x52, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x58, 0x0a, 0x0a, 0x52, + 0x6f, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, + 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfc, 0x02, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x18, - 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x22, 0xaf, 0x01, 0x0a, 0x0e, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6c, - 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, - 0x73, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, - 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, - 0x12, 0x28, 0x0a, 0x04, 0x73, 0x65, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, - 0x53, 0x65, 0x65, 0x64, 0x52, 0x04, 0x73, 0x65, 0x65, 0x64, 0x22, 0x36, 0x0a, 0x0a, 0x47, 0x65, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, - 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x22, 0x6a, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x19, + 0x0a, 0x08, 0x6c, 0x69, 0x6e, 0x6b, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6c, 0x69, 0x6e, 0x6b, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, + 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x40, 0x0a, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, + 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x55, 0x0a, + 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x2e, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x51, 0x0a, 0x05, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x10, 0x0a, + 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, + 0x10, 0x0a, 0x03, 0x77, 0x77, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x77, 0x77, + 0x77, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x70, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x69, 0x70, 0x6e, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x62, 0x70, 0x73, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x62, 0x70, 0x73, 0x22, 0x2c, 0x0a, 0x04, 0x53, 0x65, 0x65, 0x64, 0x12, + 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x63, 0x69, + 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x67, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x63, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x22, 0xaf, + 0x01, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0x4f, - 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, - 0x3f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, - 0x22, 0x25, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x22, 0x40, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, - 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x39, 0x0a, 0x0d, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, - 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, - 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x22, 0x28, 0x0a, 0x0e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x4f, - 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, - 0x9d, 0x01, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x04, 0x69, 0x74, - 0x65, 0x6d, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, - 0xf6, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x10, 0x0a, 0x03, - 0x63, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x69, 0x73, - 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x69, 0x73, 0x44, 0x69, - 0x72, 0x12, 0x2e, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, - 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x43, 0x6f, 0x75, - 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, - 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x22, 0x44, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x70, 0x66, 0x73, 0x50, - 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x69, - 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, - 0x74, 0x65, 0x6d, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x22, 0x99, 0x02, 0x0a, 0x10, 0x50, 0x75, - 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, - 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, + 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x16, + 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, + 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x28, 0x0a, 0x04, 0x73, 0x65, 0x65, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x53, 0x65, 0x65, 0x64, 0x52, 0x04, 0x73, 0x65, 0x65, 0x64, + 0x22, 0x36, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x6a, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, + 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, + 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, + 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x05, 0x6c, + 0x69, 0x6e, 0x6b, 0x73, 0x22, 0x4f, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x3f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x69, 0x6e, + 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, + 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, + 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0x25, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x22, 0x40, 0x0a, + 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, + 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, - 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x12, 0x3e, 0x0a, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x1a, 0x46, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x74, - 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, - 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x1a, 0x2f, 0x0a, 0x05, 0x43, 0x68, 0x75, - 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x95, 0x01, 0x0a, 0x11, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, - 0x74, 0x68, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, - 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x63, 0x69, - 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x4f, 0x0a, - 0x0f, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, + 0x39, 0x0a, 0x0d, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x28, - 0x0a, 0x10, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x22, 0x29, 0x0a, 0x13, 0x50, 0x75, 0x6c, 0x6c, - 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x22, 0x2c, 0x0a, 0x14, 0x50, 0x75, 0x6c, 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, - 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, - 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x22, 0x60, 0x0a, 0x0e, 0x53, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, - 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x63, 0x69, 0x64, 0x22, 0x59, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x71, - 0x0a, 0x0f, 0x4d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x28, 0x0a, 0x0e, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, + 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x4f, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x9d, 0x01, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, + 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x69, 0x74, + 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, + 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x74, + 0x65, 0x6d, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, + 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, + 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x05, + 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0xf6, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x74, + 0x65, 0x6d, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x63, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x12, 0x15, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x05, 0x69, 0x73, 0x44, 0x69, 0x72, 0x12, 0x2e, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, + 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, + 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x74, 0x65, 0x6d, + 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x74, 0x65, 0x6d, 0x73, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x69, 0x74, + 0x65, 0x6d, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, + 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x44, 0x0a, 0x14, 0x4c, 0x69, 0x73, + 0x74, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x22, + 0x99, 0x02, 0x0a, 0x10, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, + 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, + 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x48, 0x00, + 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x46, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x66, - 0x72, 0x6f, 0x6d, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x66, 0x72, 0x6f, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x6f, 0x5f, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x6f, 0x50, 0x61, 0x74, - 0x68, 0x22, 0x5a, 0x0a, 0x10, 0x4d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x65, 0x0a, - 0x11, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, - 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x72, 0x6f, 0x6f, 0x74, 0x22, 0x5c, 0x0a, 0x12, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, - 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, - 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, - 0x65, 0x64, 0x22, 0x81, 0x02, 0x0a, 0x1a, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x72, + 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x1a, + 0x2f, 0x0a, 0x05, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x95, 0x01, 0x0a, 0x11, + 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, + 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, + 0x6e, 0x65, 0x64, 0x22, 0x4f, 0x0a, 0x0f, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x22, 0x28, 0x0a, 0x10, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x75, 0x6e, + 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x22, 0x29, + 0x0a, 0x13, 0x50, 0x75, 0x6c, 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x2c, 0x0a, 0x14, 0x50, 0x75, 0x6c, + 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x22, 0x74, 0x0a, 0x0e, 0x53, 0x65, 0x74, 0x50, 0x61, + 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, + 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x63, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x63, 0x69, 0x64, 0x22, 0x59, 0x0a, + 0x0f, 0x53, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x85, 0x01, 0x0a, 0x0f, 0x4d, 0x6f, 0x76, + 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, + 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, + 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x72, + 0x6f, 0x6d, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, + 0x72, 0x6f, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x6f, 0x5f, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x6f, 0x50, 0x61, 0x74, 0x68, + 0x22, 0x5a, 0x0a, 0x10, 0x4d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x65, 0x0a, 0x11, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, - 0x4b, 0x0a, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x72, + 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x22, 0x5c, 0x0a, 0x12, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, + 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, + 0x6e, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, + 0x64, 0x22, 0x95, 0x02, 0x0a, 0x1a, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, + 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, + 0x6f, 0x74, 0x12, 0x4b, 0x0a, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x35, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, + 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x6f, + 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x1a, + 0x58, 0x0a, 0x0a, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x34, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, - 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, - 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x6f, 0x6c, 0x65, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x1a, 0x58, 0x0a, 0x0a, - 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, - 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x65, 0x0a, 0x1b, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, - 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x5a, 0x0a, - 0x1a, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, - 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, + 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x65, 0x0a, 0x1b, 0x50, 0x75, 0x73, + 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, + 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, + 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, + 0x22, 0x5a, 0x0a, 0x1a, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xc5, 0x01, 0x0a, + 0x1b, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, + 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x05, + 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, + 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x1a, 0x58, 0x0a, 0x0a, 0x52, 0x6f, + 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x41, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7b, 0x0a, 0x13, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xc5, 0x01, 0x0a, 0x1b, 0x50, 0x75, - 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x72, 0x6f, 0x6c, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, - 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, - 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x1a, 0x58, 0x0a, 0x0a, 0x52, 0x6f, 0x6c, 0x65, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x2a, 0x88, 0x01, 0x0a, 0x0e, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x52, 0x6f, 0x6c, 0x65, 0x12, 0x20, 0x0a, 0x1c, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x41, 0x43, 0x43, - 0x45, 0x53, 0x53, 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, - 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x41, - 0x43, 0x43, 0x45, 0x53, 0x53, 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x45, - 0x52, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x41, 0x43, 0x43, 0x45, - 0x53, 0x53, 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x52, 0x10, 0x02, - 0x12, 0x1a, 0x0a, 0x16, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x41, 0x43, 0x43, 0x45, 0x53, 0x53, 0x5f, - 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x41, 0x44, 0x4d, 0x49, 0x4e, 0x10, 0x03, 0x32, 0x8c, 0x0a, 0x0a, - 0x0a, 0x41, 0x50, 0x49, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x49, 0x0a, 0x06, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x1a, 0x2e, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6f, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x6e, 0x66, + 0x6f, 0x22, 0x46, 0x0a, 0x14, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x06, 0x62, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x22, 0x53, 0x0a, 0x13, 0x50, 0x75, 0x6c, + 0x6c, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x2a, + 0x0a, 0x14, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x2a, 0x88, 0x01, 0x0a, 0x0e, 0x50, + 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x12, 0x20, 0x0a, + 0x1c, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x41, 0x43, 0x43, 0x45, 0x53, 0x53, 0x5f, 0x52, 0x4f, 0x4c, + 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x1b, 0x0a, 0x17, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x41, 0x43, 0x43, 0x45, 0x53, 0x53, 0x5f, 0x52, + 0x4f, 0x4c, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, + 0x50, 0x41, 0x54, 0x48, 0x5f, 0x41, 0x43, 0x43, 0x45, 0x53, 0x53, 0x5f, 0x52, 0x4f, 0x4c, 0x45, + 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x52, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x50, 0x41, 0x54, + 0x48, 0x5f, 0x41, 0x43, 0x43, 0x45, 0x53, 0x53, 0x5f, 0x52, 0x4f, 0x4c, 0x45, 0x5f, 0x41, 0x44, + 0x4d, 0x49, 0x4e, 0x10, 0x03, 0x32, 0xc6, 0x0b, 0x0a, 0x0a, 0x41, 0x50, 0x49, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x49, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1d, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x40, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, + 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x4f, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x47, - 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4c, - 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x69, 0x73, - 0x74, 0x12, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, + 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, - 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, - 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, - 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x08, 0x4c, 0x69, 0x73, - 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x0c, 0x4c, 0x69, - 0x73, 0x74, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, + 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x43, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x56, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x50, - 0x61, 0x74, 0x68, 0x73, 0x12, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, - 0x51, 0x0a, 0x08, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x2e, 0x61, 0x70, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, + 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1f, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x70, 0x66, 0x73, 0x50, + 0x61, 0x74, 0x68, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, + 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x70, + 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x56, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x20, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, + 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x08, 0x50, 0x75, 0x6c, 0x6c, + 0x50, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x5d, 0x0a, 0x0c, 0x50, + 0x75, 0x6c, 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, - 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, - 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x30, 0x01, 0x12, 0x5d, 0x0a, 0x0c, 0x50, 0x75, 0x6c, 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, - 0x74, 0x68, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, - 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x49, 0x70, 0x66, - 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, - 0x01, 0x12, 0x4c, 0x0a, 0x07, 0x53, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1e, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x53, 0x65, - 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x53, 0x65, - 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x4f, 0x0a, 0x08, 0x4d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x6f, 0x76, - 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x6f, + 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x49, 0x70, 0x66, 0x73, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x07, 0x53, 0x65, + 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x53, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x53, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x08, 0x4d, 0x6f, 0x76, 0x65, + 0x50, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0a, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, + 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, + 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x55, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x21, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x73, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x13, 0x50, 0x75, 0x73, 0x68, 0x50, - 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x2a, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, - 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, - 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, - 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x13, 0x50, 0x75, 0x6c, - 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, - 0x12, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, - 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x30, 0x5a, 0x2e, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x78, 0x74, 0x69, 0x6c, - 0x65, 0x69, 0x6f, 0x2f, 0x67, 0x6f, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2f, 0x61, - 0x70, 0x69, 0x2f, 0x70, 0x62, 0x2f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x70, 0x0a, 0x13, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, + 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, + 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x70, 0x0a, 0x13, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, + 0x61, 0x74, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x41, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, + 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x50, + 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x5b, 0x0a, 0x0c, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x62, 0x2e, + 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x61, 0x74, 0x68, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x30, + 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x78, + 0x74, 0x69, 0x6c, 0x65, 0x69, 0x6f, 0x2f, 0x67, 0x6f, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x2f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2722,7 +3036,7 @@ func file_api_pb_buckets_buckets_proto_rawDescGZIP() []byte { } var file_api_pb_buckets_buckets_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_api_pb_buckets_buckets_proto_msgTypes = make([]protoimpl.MessageInfo, 41) +var file_api_pb_buckets_buckets_proto_msgTypes = make([]protoimpl.MessageInfo, 45) var file_api_pb_buckets_buckets_proto_goTypes = []interface{}{ (PathAccessRole)(0), // 0: api.pb.buckets.PathAccessRole (*Metadata)(nil), // 1: api.pb.buckets.Metadata @@ -2760,16 +3074,20 @@ var file_api_pb_buckets_buckets_proto_goTypes = []interface{}{ (*PushPathAccessRolesResponse)(nil), // 33: api.pb.buckets.PushPathAccessRolesResponse (*PullPathAccessRolesRequest)(nil), // 34: api.pb.buckets.PullPathAccessRolesRequest (*PullPathAccessRolesResponse)(nil), // 35: api.pb.buckets.PullPathAccessRolesResponse - nil, // 36: api.pb.buckets.Metadata.RolesEntry - nil, // 37: api.pb.buckets.Bucket.MetadataEntry - (*PushPathsRequest_Header)(nil), // 38: api.pb.buckets.PushPathsRequest.Header - (*PushPathsRequest_Chunk)(nil), // 39: api.pb.buckets.PushPathsRequest.Chunk - nil, // 40: api.pb.buckets.PushPathAccessRolesRequest.RolesEntry - nil, // 41: api.pb.buckets.PullPathAccessRolesResponse.RolesEntry + (*PushPathInfoRequest)(nil), // 36: api.pb.buckets.PushPathInfoRequest + (*PushPathInfoResponse)(nil), // 37: api.pb.buckets.PushPathInfoResponse + (*PullPathInfoRequest)(nil), // 38: api.pb.buckets.PullPathInfoRequest + (*PullPathInfoResponse)(nil), // 39: api.pb.buckets.PullPathInfoResponse + nil, // 40: api.pb.buckets.Metadata.RolesEntry + nil, // 41: api.pb.buckets.Bucket.MetadataEntry + (*PushPathsRequest_Header)(nil), // 42: api.pb.buckets.PushPathsRequest.Header + (*PushPathsRequest_Chunk)(nil), // 43: api.pb.buckets.PushPathsRequest.Chunk + nil, // 44: api.pb.buckets.PushPathAccessRolesRequest.RolesEntry + nil, // 45: api.pb.buckets.PullPathAccessRolesResponse.RolesEntry } var file_api_pb_buckets_buckets_proto_depIdxs = []int32{ - 36, // 0: api.pb.buckets.Metadata.roles:type_name -> api.pb.buckets.Metadata.RolesEntry - 37, // 1: api.pb.buckets.Bucket.metadata:type_name -> api.pb.buckets.Bucket.MetadataEntry + 40, // 0: api.pb.buckets.Metadata.roles:type_name -> api.pb.buckets.Metadata.RolesEntry + 41, // 1: api.pb.buckets.Bucket.metadata:type_name -> api.pb.buckets.Bucket.MetadataEntry 2, // 2: api.pb.buckets.CreateResponse.bucket:type_name -> api.pb.buckets.Bucket 3, // 3: api.pb.buckets.CreateResponse.links:type_name -> api.pb.buckets.Links 4, // 4: api.pb.buckets.CreateResponse.seed:type_name -> api.pb.buckets.Seed @@ -2783,54 +3101,59 @@ var file_api_pb_buckets_buckets_proto_depIdxs = []int32{ 17, // 12: api.pb.buckets.PathItem.items:type_name -> api.pb.buckets.PathItem 1, // 13: api.pb.buckets.PathItem.metadata:type_name -> api.pb.buckets.Metadata 17, // 14: api.pb.buckets.ListIpfsPathResponse.item:type_name -> api.pb.buckets.PathItem - 38, // 15: api.pb.buckets.PushPathsRequest.header:type_name -> api.pb.buckets.PushPathsRequest.Header - 39, // 16: api.pb.buckets.PushPathsRequest.chunk:type_name -> api.pb.buckets.PushPathsRequest.Chunk + 42, // 15: api.pb.buckets.PushPathsRequest.header:type_name -> api.pb.buckets.PushPathsRequest.Header + 43, // 16: api.pb.buckets.PushPathsRequest.chunk:type_name -> api.pb.buckets.PushPathsRequest.Chunk 2, // 17: api.pb.buckets.PushPathsResponse.bucket:type_name -> api.pb.buckets.Bucket 2, // 18: api.pb.buckets.SetPathResponse.bucket:type_name -> api.pb.buckets.Bucket 2, // 19: api.pb.buckets.MovePathResponse.bucket:type_name -> api.pb.buckets.Bucket 2, // 20: api.pb.buckets.RemovePathResponse.bucket:type_name -> api.pb.buckets.Bucket - 40, // 21: api.pb.buckets.PushPathAccessRolesRequest.roles:type_name -> api.pb.buckets.PushPathAccessRolesRequest.RolesEntry + 44, // 21: api.pb.buckets.PushPathAccessRolesRequest.roles:type_name -> api.pb.buckets.PushPathAccessRolesRequest.RolesEntry 2, // 22: api.pb.buckets.PushPathAccessRolesResponse.bucket:type_name -> api.pb.buckets.Bucket - 41, // 23: api.pb.buckets.PullPathAccessRolesResponse.roles:type_name -> api.pb.buckets.PullPathAccessRolesResponse.RolesEntry - 0, // 24: api.pb.buckets.Metadata.RolesEntry.value:type_name -> api.pb.buckets.PathAccessRole - 1, // 25: api.pb.buckets.Bucket.MetadataEntry.value:type_name -> api.pb.buckets.Metadata - 0, // 26: api.pb.buckets.PushPathAccessRolesRequest.RolesEntry.value:type_name -> api.pb.buckets.PathAccessRole - 0, // 27: api.pb.buckets.PullPathAccessRolesResponse.RolesEntry.value:type_name -> api.pb.buckets.PathAccessRole - 5, // 28: api.pb.buckets.APIService.Create:input_type -> api.pb.buckets.CreateRequest - 7, // 29: api.pb.buckets.APIService.Get:input_type -> api.pb.buckets.GetRequest - 9, // 30: api.pb.buckets.APIService.GetLinks:input_type -> api.pb.buckets.GetLinksRequest - 11, // 31: api.pb.buckets.APIService.List:input_type -> api.pb.buckets.ListRequest - 13, // 32: api.pb.buckets.APIService.Remove:input_type -> api.pb.buckets.RemoveRequest - 15, // 33: api.pb.buckets.APIService.ListPath:input_type -> api.pb.buckets.ListPathRequest - 18, // 34: api.pb.buckets.APIService.ListIpfsPath:input_type -> api.pb.buckets.ListIpfsPathRequest - 20, // 35: api.pb.buckets.APIService.PushPaths:input_type -> api.pb.buckets.PushPathsRequest - 22, // 36: api.pb.buckets.APIService.PullPath:input_type -> api.pb.buckets.PullPathRequest - 24, // 37: api.pb.buckets.APIService.PullIpfsPath:input_type -> api.pb.buckets.PullIpfsPathRequest - 26, // 38: api.pb.buckets.APIService.SetPath:input_type -> api.pb.buckets.SetPathRequest - 28, // 39: api.pb.buckets.APIService.MovePath:input_type -> api.pb.buckets.MovePathRequest - 30, // 40: api.pb.buckets.APIService.RemovePath:input_type -> api.pb.buckets.RemovePathRequest - 32, // 41: api.pb.buckets.APIService.PushPathAccessRoles:input_type -> api.pb.buckets.PushPathAccessRolesRequest - 34, // 42: api.pb.buckets.APIService.PullPathAccessRoles:input_type -> api.pb.buckets.PullPathAccessRolesRequest - 6, // 43: api.pb.buckets.APIService.Create:output_type -> api.pb.buckets.CreateResponse - 8, // 44: api.pb.buckets.APIService.Get:output_type -> api.pb.buckets.GetResponse - 10, // 45: api.pb.buckets.APIService.GetLinks:output_type -> api.pb.buckets.GetLinksResponse - 12, // 46: api.pb.buckets.APIService.List:output_type -> api.pb.buckets.ListResponse - 14, // 47: api.pb.buckets.APIService.Remove:output_type -> api.pb.buckets.RemoveResponse - 16, // 48: api.pb.buckets.APIService.ListPath:output_type -> api.pb.buckets.ListPathResponse - 19, // 49: api.pb.buckets.APIService.ListIpfsPath:output_type -> api.pb.buckets.ListIpfsPathResponse - 21, // 50: api.pb.buckets.APIService.PushPaths:output_type -> api.pb.buckets.PushPathsResponse - 23, // 51: api.pb.buckets.APIService.PullPath:output_type -> api.pb.buckets.PullPathResponse - 25, // 52: api.pb.buckets.APIService.PullIpfsPath:output_type -> api.pb.buckets.PullIpfsPathResponse - 27, // 53: api.pb.buckets.APIService.SetPath:output_type -> api.pb.buckets.SetPathResponse - 29, // 54: api.pb.buckets.APIService.MovePath:output_type -> api.pb.buckets.MovePathResponse - 31, // 55: api.pb.buckets.APIService.RemovePath:output_type -> api.pb.buckets.RemovePathResponse - 33, // 56: api.pb.buckets.APIService.PushPathAccessRoles:output_type -> api.pb.buckets.PushPathAccessRolesResponse - 35, // 57: api.pb.buckets.APIService.PullPathAccessRoles:output_type -> api.pb.buckets.PullPathAccessRolesResponse - 43, // [43:58] is the sub-list for method output_type - 28, // [28:43] is the sub-list for method input_type - 28, // [28:28] is the sub-list for extension type_name - 28, // [28:28] is the sub-list for extension extendee - 0, // [0:28] is the sub-list for field type_name + 45, // 23: api.pb.buckets.PullPathAccessRolesResponse.roles:type_name -> api.pb.buckets.PullPathAccessRolesResponse.RolesEntry + 2, // 24: api.pb.buckets.PushPathInfoResponse.bucket:type_name -> api.pb.buckets.Bucket + 0, // 25: api.pb.buckets.Metadata.RolesEntry.value:type_name -> api.pb.buckets.PathAccessRole + 1, // 26: api.pb.buckets.Bucket.MetadataEntry.value:type_name -> api.pb.buckets.Metadata + 0, // 27: api.pb.buckets.PushPathAccessRolesRequest.RolesEntry.value:type_name -> api.pb.buckets.PathAccessRole + 0, // 28: api.pb.buckets.PullPathAccessRolesResponse.RolesEntry.value:type_name -> api.pb.buckets.PathAccessRole + 5, // 29: api.pb.buckets.APIService.Create:input_type -> api.pb.buckets.CreateRequest + 7, // 30: api.pb.buckets.APIService.Get:input_type -> api.pb.buckets.GetRequest + 9, // 31: api.pb.buckets.APIService.GetLinks:input_type -> api.pb.buckets.GetLinksRequest + 11, // 32: api.pb.buckets.APIService.List:input_type -> api.pb.buckets.ListRequest + 13, // 33: api.pb.buckets.APIService.Remove:input_type -> api.pb.buckets.RemoveRequest + 15, // 34: api.pb.buckets.APIService.ListPath:input_type -> api.pb.buckets.ListPathRequest + 18, // 35: api.pb.buckets.APIService.ListIpfsPath:input_type -> api.pb.buckets.ListIpfsPathRequest + 20, // 36: api.pb.buckets.APIService.PushPaths:input_type -> api.pb.buckets.PushPathsRequest + 22, // 37: api.pb.buckets.APIService.PullPath:input_type -> api.pb.buckets.PullPathRequest + 24, // 38: api.pb.buckets.APIService.PullIpfsPath:input_type -> api.pb.buckets.PullIpfsPathRequest + 26, // 39: api.pb.buckets.APIService.SetPath:input_type -> api.pb.buckets.SetPathRequest + 28, // 40: api.pb.buckets.APIService.MovePath:input_type -> api.pb.buckets.MovePathRequest + 30, // 41: api.pb.buckets.APIService.RemovePath:input_type -> api.pb.buckets.RemovePathRequest + 32, // 42: api.pb.buckets.APIService.PushPathAccessRoles:input_type -> api.pb.buckets.PushPathAccessRolesRequest + 34, // 43: api.pb.buckets.APIService.PullPathAccessRoles:input_type -> api.pb.buckets.PullPathAccessRolesRequest + 36, // 44: api.pb.buckets.APIService.PushPathInfo:input_type -> api.pb.buckets.PushPathInfoRequest + 38, // 45: api.pb.buckets.APIService.PullPathInfo:input_type -> api.pb.buckets.PullPathInfoRequest + 6, // 46: api.pb.buckets.APIService.Create:output_type -> api.pb.buckets.CreateResponse + 8, // 47: api.pb.buckets.APIService.Get:output_type -> api.pb.buckets.GetResponse + 10, // 48: api.pb.buckets.APIService.GetLinks:output_type -> api.pb.buckets.GetLinksResponse + 12, // 49: api.pb.buckets.APIService.List:output_type -> api.pb.buckets.ListResponse + 14, // 50: api.pb.buckets.APIService.Remove:output_type -> api.pb.buckets.RemoveResponse + 16, // 51: api.pb.buckets.APIService.ListPath:output_type -> api.pb.buckets.ListPathResponse + 19, // 52: api.pb.buckets.APIService.ListIpfsPath:output_type -> api.pb.buckets.ListIpfsPathResponse + 21, // 53: api.pb.buckets.APIService.PushPaths:output_type -> api.pb.buckets.PushPathsResponse + 23, // 54: api.pb.buckets.APIService.PullPath:output_type -> api.pb.buckets.PullPathResponse + 25, // 55: api.pb.buckets.APIService.PullIpfsPath:output_type -> api.pb.buckets.PullIpfsPathResponse + 27, // 56: api.pb.buckets.APIService.SetPath:output_type -> api.pb.buckets.SetPathResponse + 29, // 57: api.pb.buckets.APIService.MovePath:output_type -> api.pb.buckets.MovePathResponse + 31, // 58: api.pb.buckets.APIService.RemovePath:output_type -> api.pb.buckets.RemovePathResponse + 33, // 59: api.pb.buckets.APIService.PushPathAccessRoles:output_type -> api.pb.buckets.PushPathAccessRolesResponse + 35, // 60: api.pb.buckets.APIService.PullPathAccessRoles:output_type -> api.pb.buckets.PullPathAccessRolesResponse + 37, // 61: api.pb.buckets.APIService.PushPathInfo:output_type -> api.pb.buckets.PushPathInfoResponse + 39, // 62: api.pb.buckets.APIService.PullPathInfo:output_type -> api.pb.buckets.PullPathInfoResponse + 46, // [46:63] is the sub-list for method output_type + 29, // [29:46] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name } func init() { file_api_pb_buckets_buckets_proto_init() } @@ -3259,8 +3582,32 @@ func file_api_pb_buckets_buckets_proto_init() { return nil } } + file_api_pb_buckets_buckets_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PushPathInfoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_pb_buckets_buckets_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PushPathInfoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } file_api_pb_buckets_buckets_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PushPathsRequest_Header); i { + switch v := v.(*PullPathInfoRequest); i { case 0: return &v.state case 1: @@ -3272,6 +3619,30 @@ func file_api_pb_buckets_buckets_proto_init() { } } file_api_pb_buckets_buckets_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PullPathInfoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_pb_buckets_buckets_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PushPathsRequest_Header); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_pb_buckets_buckets_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*PushPathsRequest_Chunk); i { case 0: return &v.state @@ -3294,7 +3665,7 @@ func file_api_pb_buckets_buckets_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_api_pb_buckets_buckets_proto_rawDesc, NumEnums: 1, - NumMessages: 41, + NumMessages: 45, NumExtensions: 0, NumServices: 1, }, @@ -3336,6 +3707,8 @@ type APIServiceClient interface { RemovePath(ctx context.Context, in *RemovePathRequest, opts ...grpc.CallOption) (*RemovePathResponse, error) PushPathAccessRoles(ctx context.Context, in *PushPathAccessRolesRequest, opts ...grpc.CallOption) (*PushPathAccessRolesResponse, error) PullPathAccessRoles(ctx context.Context, in *PullPathAccessRolesRequest, opts ...grpc.CallOption) (*PullPathAccessRolesResponse, error) + PushPathInfo(ctx context.Context, in *PushPathInfoRequest, opts ...grpc.CallOption) (*PushPathInfoResponse, error) + PullPathInfo(ctx context.Context, in *PullPathInfoRequest, opts ...grpc.CallOption) (*PullPathInfoResponse, error) } type aPIServiceClient struct { @@ -3549,6 +3922,24 @@ func (c *aPIServiceClient) PullPathAccessRoles(ctx context.Context, in *PullPath return out, nil } +func (c *aPIServiceClient) PushPathInfo(ctx context.Context, in *PushPathInfoRequest, opts ...grpc.CallOption) (*PushPathInfoResponse, error) { + out := new(PushPathInfoResponse) + err := c.cc.Invoke(ctx, "/api.pb.buckets.APIService/PushPathInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aPIServiceClient) PullPathInfo(ctx context.Context, in *PullPathInfoRequest, opts ...grpc.CallOption) (*PullPathInfoResponse, error) { + out := new(PullPathInfoResponse) + err := c.cc.Invoke(ctx, "/api.pb.buckets.APIService/PullPathInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // APIServiceServer is the server API for APIService service. type APIServiceServer interface { Create(context.Context, *CreateRequest) (*CreateResponse, error) @@ -3566,6 +3957,8 @@ type APIServiceServer interface { RemovePath(context.Context, *RemovePathRequest) (*RemovePathResponse, error) PushPathAccessRoles(context.Context, *PushPathAccessRolesRequest) (*PushPathAccessRolesResponse, error) PullPathAccessRoles(context.Context, *PullPathAccessRolesRequest) (*PullPathAccessRolesResponse, error) + PushPathInfo(context.Context, *PushPathInfoRequest) (*PushPathInfoResponse, error) + PullPathInfo(context.Context, *PullPathInfoRequest) (*PullPathInfoResponse, error) } // UnimplementedAPIServiceServer can be embedded to have forward compatible implementations. @@ -3617,6 +4010,12 @@ func (*UnimplementedAPIServiceServer) PushPathAccessRoles(context.Context, *Push func (*UnimplementedAPIServiceServer) PullPathAccessRoles(context.Context, *PullPathAccessRolesRequest) (*PullPathAccessRolesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method PullPathAccessRoles not implemented") } +func (*UnimplementedAPIServiceServer) PushPathInfo(context.Context, *PushPathInfoRequest) (*PushPathInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PushPathInfo not implemented") +} +func (*UnimplementedAPIServiceServer) PullPathInfo(context.Context, *PullPathInfoRequest) (*PullPathInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PullPathInfo not implemented") +} func RegisterAPIServiceServer(s *grpc.Server, srv APIServiceServer) { s.RegisterService(&_APIService_serviceDesc, srv) @@ -3906,6 +4305,42 @@ func _APIService_PullPathAccessRoles_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _APIService_PushPathInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PushPathInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(APIServiceServer).PushPathInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.pb.buckets.APIService/PushPathInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(APIServiceServer).PushPathInfo(ctx, req.(*PushPathInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _APIService_PullPathInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PullPathInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(APIServiceServer).PullPathInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.pb.buckets.APIService/PullPathInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(APIServiceServer).PullPathInfo(ctx, req.(*PullPathInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _APIService_serviceDesc = grpc.ServiceDesc{ ServiceName: "api.pb.buckets.APIService", HandlerType: (*APIServiceServer)(nil), @@ -3958,6 +4393,14 @@ var _APIService_serviceDesc = grpc.ServiceDesc{ MethodName: "PullPathAccessRoles", Handler: _APIService_PullPathAccessRoles_Handler, }, + { + MethodName: "PushPathInfo", + Handler: _APIService_PushPathInfo_Handler, + }, + { + MethodName: "PullPathInfo", + Handler: _APIService_PullPathInfo_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/api/pb/buckets/buckets.proto b/api/pb/buckets/buckets.proto index 0f7dfc8..f2a4918 100644 --- a/api/pb/buckets/buckets.proto +++ b/api/pb/buckets/buckets.proto @@ -3,28 +3,30 @@ package api.pb.buckets; option go_package = "github.com/textileio/go-buckets/api/pb/buckets"; message Metadata { - string key = 1; - map roles = 2; - int64 updated_at = 3; + string key = 1; + map roles = 2; + bytes info = 3; + int64 updated_at = 4; } message Bucket { - string thread = 1; - string key = 2; - string owner = 3; - string name = 4; - int32 version = 5; - string link_key = 6; - string path = 7; - map metadata = 8; - int64 created_at = 9; - int64 updated_at = 10; + string thread = 1; + string key = 2; + string owner = 3; + string name = 4; + int32 version = 5; + string link_key = 6; + string path = 7; + map metadata = 8; + int64 created_at = 9; + int64 updated_at = 10; } message Links { string url = 1; string www = 2; string ipns = 3; + string bps = 4; } message Seed { @@ -161,8 +163,9 @@ message PullIpfsPathResponse { message SetPathRequest { string thread = 1; string key = 2; - string path = 3; - string cid = 4; + string root = 3; + string path = 4; + string cid = 5; } message SetPathResponse { @@ -173,8 +176,9 @@ message SetPathResponse { message MovePathRequest { string thread = 1; string key = 2; - string from_path = 3; - string to_path = 4; + string root = 3; + string from_path = 4; + string to_path = 5; } message MovePathResponse { @@ -185,8 +189,8 @@ message MovePathResponse { message RemovePathRequest { string thread = 1; string key = 2; - string path = 3; - string root = 4; + string root = 3; + string path = 4; } message RemovePathResponse { @@ -205,7 +209,8 @@ message PushPathAccessRolesRequest { string thread = 1; string key = 2; string path = 3; - map roles = 4; + string root = 4; + map roles = 5; } message PushPathAccessRolesResponse { @@ -223,6 +228,28 @@ message PullPathAccessRolesResponse { map roles = 1; } +message PushPathInfoRequest { + string thread = 1; + string key = 2; + string path = 3; + string root = 4; + bytes info = 5; +} + +message PushPathInfoResponse { + Bucket bucket = 1; +} + +message PullPathInfoRequest { + string thread = 1; + string key = 2; + string path = 3; +} + +message PullPathInfoResponse { + bytes info = 1; +} + service APIService { rpc Create(CreateRequest) returns (CreateResponse) {} rpc Get(GetRequest) returns (GetResponse) {} @@ -241,4 +268,7 @@ service APIService { rpc PushPathAccessRoles(PushPathAccessRolesRequest) returns (PushPathAccessRolesResponse) {} rpc PullPathAccessRoles(PullPathAccessRolesRequest) returns (PullPathAccessRolesResponse) {} + + rpc PushPathInfo(PushPathInfoRequest) returns (PushPathInfoResponse) {} + rpc PullPathInfo(PullPathInfoRequest) returns (PullPathInfoResponse) {} } diff --git a/api/service.go b/api/service.go index 9b01c62..05be0c6 100644 --- a/api/service.go +++ b/api/service.go @@ -11,7 +11,7 @@ import ( "github.com/textileio/go-buckets" "github.com/textileio/go-buckets/api/cast" pb "github.com/textileio/go-buckets/api/pb/buckets" - "github.com/textileio/go-buckets/util" + "github.com/textileio/go-buckets/dag" "github.com/textileio/go-threads/core/did" core "github.com/textileio/go-threads/core/thread" ) @@ -57,7 +57,7 @@ func (s *Service) Create(ctx context.Context, req *pb.CreateRequest) (*pb.Create if err != nil { return nil, err } - links, err := s.lib.GetLinksForBucket(ctx, bucket, "", identity) + links, err := s.lib.GetLinksForBucket(ctx, bucket, identity, "") if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (s *Service) Get(ctx context.Context, req *pb.GetRequest) (*pb.GetResponse, if err != nil { return nil, err } - links, err := s.lib.GetLinksForBucket(ctx, bucket, "", identity) + links, err := s.lib.GetLinksForBucket(ctx, bucket, identity, "") if err != nil { return nil, err } @@ -98,7 +98,7 @@ func (s *Service) GetLinks(ctx context.Context, req *pb.GetLinksRequest) (*pb.Ge return nil, err } - links, err := s.lib.GetLinks(ctx, thread, req.Key, req.Path, identity) + links, err := s.lib.GetLinks(ctx, thread, req.Key, identity, req.Path) if err != nil { return nil, err } @@ -147,11 +147,11 @@ func (s *Service) ListPath(ctx context.Context, req *pb.ListPathRequest) (*pb.Li return nil, err } - item, bucket, err := s.lib.ListPath(ctx, thread, req.Key, req.Path, identity) + item, bucket, err := s.lib.ListPath(ctx, thread, req.Key, identity, req.Path) if err != nil { return nil, err } - links, err := s.lib.GetLinksForBucket(ctx, bucket, req.Path, identity) + links, err := s.lib.GetLinksForBucket(ctx, bucket, identity, req.Path) if err != nil { return nil, err } @@ -194,26 +194,24 @@ func (s *Service) PushPaths(server pb.APIService_PushPathsServer) error { return fmt.Errorf("decoding thread: %v", err) } key = payload.Header.Key - if len(payload.Header.Root) != 0 { - root, err = util.NewResolvedPath(payload.Header.Root) - if err != nil { - return fmt.Errorf("resolving root path: %v", err) - } + root, err = rootFromString(payload.Header.Root) + if err != nil { + return err } default: return fmt.Errorf("push bucket path header is required") } - in, out, errs := s.lib.PushPaths(server.Context(), thread, key, root, identity) + in, out, errs := s.lib.PushPaths(server.Context(), thread, key, identity, root) if len(errs) != 0 { return <-errs } errCh := make(chan error) go func() { + defer close(in) for { req, err := server.Recv() if err == io.EOF { - close(in) return } else if err != nil { errCh <- fmt.Errorf("on receive: %v", err) @@ -221,9 +219,9 @@ func (s *Service) PushPaths(server pb.APIService_PushPathsServer) error { } switch payload := req.Payload.(type) { case *pb.PushPathsRequest_Chunk_: - in <- buckets.PushPathsChunk{ - Path: payload.Chunk.Path, - Data: payload.Chunk.Data, + in <- buckets.PushPathsInput{ + Path: payload.Chunk.Path, + Chunk: payload.Chunk.Data, } default: errCh <- fmt.Errorf("invalid request") @@ -258,7 +256,7 @@ func (s *Service) PullPath(req *pb.PullPathRequest, server pb.APIService_PullPat return err } - reader, err := s.lib.PullPath(server.Context(), thread, req.Key, req.Path, identity) + reader, err := s.lib.PullPath(server.Context(), thread, req.Key, identity, req.Path) if err != nil { return err } @@ -312,12 +310,16 @@ func (s *Service) SetPath(ctx context.Context, req *pb.SetPathRequest) (*pb.SetP if err != nil { return nil, err } + root, err := rootFromString(req.Root) + if err != nil { + return nil, err + } cid, err := c.Decode(req.Cid) if err != nil { return nil, fmt.Errorf("decoding cid: %v", err) } - pinned, bucket, err := s.lib.SetPath(ctx, thread, req.Key, req.Path, cid, identity) + pinned, bucket, err := s.lib.SetPath(ctx, thread, req.Key, identity, root, req.Path, cid, nil) if err != nil { return nil, err } @@ -333,8 +335,12 @@ func (s *Service) MovePath(ctx context.Context, req *pb.MovePathRequest) (res *p if err != nil { return nil, err } + root, err := rootFromString(req.Root) + if err != nil { + return nil, err + } - pinned, bucket, err := s.lib.MovePath(ctx, thread, req.Key, req.FromPath, req.ToPath, identity) + pinned, bucket, err := s.lib.MovePath(ctx, thread, req.Key, identity, root, req.FromPath, req.ToPath) if err != nil { return nil, err } @@ -349,15 +355,12 @@ func (s *Service) RemovePath(ctx context.Context, req *pb.RemovePathRequest) (re if err != nil { return nil, err } - var root path.Resolved - if len(req.Root) != 0 { - root, err = util.NewResolvedPath(req.Root) - if err != nil { - return nil, fmt.Errorf("resolving root path: %v", err) - } + root, err := rootFromString(req.Root) + if err != nil { + return nil, err } - pinned, bucket, err := s.lib.RemovePath(ctx, thread, req.Key, req.Path, root, identity) + pinned, bucket, err := s.lib.RemovePath(ctx, thread, req.Key, identity, root, req.Path) if err != nil { return nil, err } @@ -375,9 +378,13 @@ func (s *Service) PushPathAccessRoles( if err != nil { return nil, err } + root, err := rootFromString(req.Root) + if err != nil { + return nil, err + } roles := cast.RolesFromPb(req.Roles) - pinned, bucket, err := s.lib.PushPathAccessRoles(ctx, thread, req.Key, req.Path, roles, identity) + pinned, bucket, err := s.lib.PushPathAccessRoles(ctx, thread, req.Key, identity, root, req.Path, roles) if err != nil { return nil, err } @@ -396,7 +403,7 @@ func (s *Service) PullPathAccessRoles( return nil, err } - roles, err := s.lib.PullPathAccessRoles(ctx, thread, req.Key, req.Path, identity) + roles, err := s.lib.PullPathAccessRoles(ctx, thread, req.Key, identity, req.Path) if err != nil { return nil, err } @@ -405,6 +412,55 @@ func (s *Service) PullPathAccessRoles( }, nil } +func (s *Service) PushPathInfo( + ctx context.Context, + req *pb.PushPathInfoRequest, +) (res *pb.PushPathInfoResponse, err error) { + thread, identity, err := getThreadAndIdentity(ctx, req.Thread) + if err != nil { + return nil, err + } + root, err := rootFromString(req.Root) + if err != nil { + return nil, err + } + info, err := cast.InfoFromPb(req.Info) + if err != nil { + return nil, err + } + + bucket, err := s.lib.PushPathInfo(ctx, thread, req.Key, identity, root, req.Path, info) + if err != nil { + return nil, err + } + return &pb.PushPathInfoResponse{ + Bucket: cast.BucketToPb(bucket), + }, nil +} + +func (s *Service) PullPathInfo( + ctx context.Context, + req *pb.PullPathInfoRequest, +) (*pb.PullPathInfoResponse, error) { + thread, identity, err := getThreadAndIdentity(ctx, req.Thread) + if err != nil { + return nil, err + } + + info, err := s.lib.PullPathInfo(ctx, thread, req.Key, identity, req.Path) + if err != nil { + return nil, err + } + data, err := cast.InfoToPb(info) + if err != nil { + return nil, err + } + + return &pb.PullPathInfoResponse{ + Info: data, + }, nil +} + func getThreadAndIdentity(ctx context.Context, threadStr string) (thread core.ID, identity did.Token, err error) { if len(threadStr) != 0 { thread, err = core.Decode(threadStr) @@ -418,3 +474,13 @@ func getThreadAndIdentity(ctx context.Context, threadStr string) (thread core.ID } return thread, identity, nil } + +func rootFromString(root string) (r path.Resolved, err error) { + if len(root) != 0 { + r, err = dag.NewResolvedPath(root) + if err != nil { + return nil, fmt.Errorf("resolving root path: %v", err) + } + } + return r, nil +} diff --git a/buckets.go b/buckets.go index 5f941bb..5e0ddb9 100644 --- a/buckets.go +++ b/buckets.go @@ -1,9 +1,6 @@ package buckets -// @todo: Validate all thread IDs -// @todo: Validate all identities // @todo: Clean up error messages -// @todo: Enforce fast-forward-only in SetPath and MovePath, PushPathAccessRoles import ( "context" @@ -20,12 +17,12 @@ import ( "github.com/textileio/go-buckets/dag" "github.com/textileio/go-buckets/dns" "github.com/textileio/go-buckets/ipns" - "github.com/textileio/go-buckets/util" dbc "github.com/textileio/go-threads/api/client" "github.com/textileio/go-threads/core/did" core "github.com/textileio/go-threads/core/thread" "github.com/textileio/go-threads/db" nc "github.com/textileio/go-threads/net/api/client" + "github.com/textileio/go-threads/net/util" nutil "github.com/textileio/go-threads/net/util" ) @@ -35,6 +32,9 @@ var ( // GatewayURL is used to construct externally facing bucket links. GatewayURL string + // ThreadsGatewayURL is used to construct externally facing bucket links. + ThreadsGatewayURL string + // WWWDomain can be set to specify the domain to use for bucket website hosting, e.g., // if this is set to mydomain.com, buckets can be rendered as a website at the following URL: // https://.mydomain.com @@ -46,6 +46,18 @@ var ( movePathRegexp = regexp.MustCompile("/ipfs/([^/]+)/") ) +// IsPathNotFoundErr returns whether or not an error indicates a bucket path was not found. +// This is needed because public and private (encrypted) buckets can return different +// errors for the same operation. +func IsPathNotFoundErr(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "could not resolve path") || + strings.Contains(msg, "no link named") +} + // Bucket adds thread ID to collection.Bucket. type Bucket struct { Thread core.ID `json:"thread"` @@ -72,6 +84,8 @@ type Links struct { WWW string `json:"www"` // IPNS is the bucket IPNS address. IPNS string `json:"ipns"` + // BPS is the bucket pinning service URL. + BPS string `json:"bps"` } // Seed describes a bucket seed file. @@ -101,6 +115,21 @@ func (l lock) Key() string { return string(l) } +// Txn allows for holding a bucket lock while performing multiple write operations. +type Txn struct { + b *Buckets + thread core.ID + key string + identity did.Token + lock *util.Semaphore +} + +// Close the Txn, releasing the bucket for additional writes. +func (t *Txn) Close() error { + t.lock.Release() + return nil +} + // NewBuckets returns a new buckets library. func NewBuckets( net *nc.Client, @@ -130,15 +159,42 @@ func (b *Buckets) Close() error { return nil } +// Net returns the underlying thread net client. func (b *Buckets) Net() *nc.Client { return b.net } +// DB returns the underlying thread db client. func (b *Buckets) DB() *dbc.Client { return b.db } +// Ipfs returns the underlying IPFS client. +func (b *Buckets) Ipfs() iface.CoreAPI { + return b.ipfs +} + +// NewTxn returns a new Txn for bucket key. +func (b *Buckets) NewTxn(thread core.ID, key string, identity did.Token) (*Txn, error) { + if err := thread.Validate(); err != nil { + return nil, fmt.Errorf("invalid thread id: %v", err) + } + lk := b.locks.Get(lock(key)) + lk.Acquire() + return &Txn{ + b: b, + thread: thread, + key: key, + identity: identity, + lock: lk, + }, nil +} + +// Get returns a bucket by thread id and key. func (b *Buckets) Get(ctx context.Context, thread core.ID, key string, identity did.Token) (*Bucket, error) { + if err := thread.Validate(); err != nil { + return nil, fmt.Errorf("invalid thread id: %v", err) + } instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) if err != nil { return nil, err @@ -147,27 +203,33 @@ func (b *Buckets) Get(ctx context.Context, thread core.ID, key string, identity return instanceToBucket(thread, instance), nil } +// GetLinks returns a Links object containing the bucket thread, IPNS, and WWW links by thread and bucket key. func (b *Buckets) GetLinks( ctx context.Context, thread core.ID, - key, pth string, + key string, identity did.Token, + pth string, ) (links Links, err error) { + if err := thread.Validate(); err != nil { + return links, fmt.Errorf("invalid thread id: %v", err) + } instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) if err != nil { return links, err } log.Debugf("got %s links", key) - return b.GetLinksForBucket(ctx, instanceToBucket(thread, instance), pth, identity) + return b.GetLinksForBucket(ctx, instanceToBucket(thread, instance), identity, pth) } +// GetLinksForBucket returns a Links object containing the bucket thread, IPNS, and WWW links. func (b *Buckets) GetLinksForBucket( ctx context.Context, bucket *Bucket, - pth string, identity did.Token, + pth string, ) (links Links, err error) { - links.URL = fmt.Sprintf("%s/thread/%s/%s/%s", GatewayURL, bucket.Thread, collection.Name, bucket.Key) + links.URL = fmt.Sprintf("%s/thread/%s/%s/%s", ThreadsGatewayURL, bucket.Thread, collection.Name, bucket.Key) if len(WWWDomain) != 0 { parts := strings.Split(GatewayURL, "://") if len(parts) < 2 { @@ -188,7 +250,7 @@ func (b *Buckets) GetLinksForBucket( } linkKey := bucket.GetLinkEncryptionKey() if _, err := dag.GetNodeAtPath(ctx, b.ipfs, npth, linkKey); err != nil { - return links, err + return links, fmt.Errorf("could not resolve path: %s", pth) } pth = "/" + pth links.URL += pth @@ -196,19 +258,25 @@ func (b *Buckets) GetLinksForBucket( links.WWW += pth } links.IPNS += pth + } else { + links.BPS = fmt.Sprintf("%s/bps/%s", GatewayURL, bucket.Key) } - if bucket.IsPrivate() { - query := "?token=" + string(identity) - links.URL += query - if len(links.WWW) != 0 { - links.WWW += query - } - links.IPNS += query + + query := "?token=" + string(identity) + links.URL += query + if len(links.WWW) != 0 { + links.WWW += query } + links.IPNS += query + return links, nil } +// List returns all buckets under a thread. func (b *Buckets) List(ctx context.Context, thread core.ID, identity did.Token) ([]Bucket, error) { + if err := thread.Validate(); err != nil { + return nil, fmt.Errorf("invalid thread id: %v", err) + } list, err := b.c.List(ctx, thread, &db.Query{}, &collection.Bucket{}, collection.WithIdentity(identity)) if err != nil { return nil, fmt.Errorf("listing buckets: %v", err) @@ -224,48 +292,55 @@ func (b *Buckets) List(ctx context.Context, thread core.ID, identity did.Token) return bucks, nil } +// Remove removes an entire bucket, unpinning all data. func (b *Buckets) Remove(ctx context.Context, thread core.ID, key string, identity did.Token) (int64, error) { - lk := b.locks.Get(lock(key)) - lk.Acquire() - defer lk.Release() + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + return 0, err + } + defer txn.Close() + return txn.Remove(ctx) +} - instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) +// Remove is Txn based Remove. +func (t *Txn) Remove(ctx context.Context) (int64, error) { + instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) if err != nil { return 0, err } - if err := b.c.Delete(ctx, thread, key, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Delete(ctx, t.thread, t.key, collection.WithIdentity(t.identity)); err != nil { return 0, fmt.Errorf("deleting bucket: %v", err) } - buckPath, err := util.NewResolvedPath(instance.Path) + buckPath, err := dag.NewResolvedPath(instance.Path) if err != nil { return 0, fmt.Errorf("resolving path: %v", err) } linkKey := instance.GetLinkEncryptionKey() if linkKey != nil { - ctx, err = dag.UnpinNodeAndBranch(ctx, b.ipfs, buckPath, linkKey) + ctx, err = dag.UnpinNodeAndBranch(ctx, t.b.ipfs, buckPath, linkKey) if err != nil { return 0, err } } else { - ctx, err = dag.UnpinPath(ctx, b.ipfs, buckPath) + ctx, err = dag.UnpinPath(ctx, t.b.ipfs, buckPath) if err != nil { return 0, err } } - if err := b.ipns.RemoveKey(ctx, key); err != nil { + if err := t.b.ipns.RemoveKey(ctx, t.key); err != nil { return 0, err } - log.Debugf("removed %s", key) + log.Debugf("removed %s", t.key) return dag.GetPinnedBytes(ctx), nil } func (b *Buckets) saveAndPublish( ctx context.Context, thread core.ID, - instance *collection.Bucket, identity did.Token, + instance *collection.Bucket, ) error { if err := b.c.Save(ctx, thread, instance, collection.WithIdentity(identity)); err != nil { return fmt.Errorf("saving bucket: %v", err) diff --git a/cmd/buck/cli/cli.go b/cmd/buck/cli/cli.go index 623e7ab..4079e64 100644 --- a/cmd/buck/cli/cli.go +++ b/cmd/buck/cli/cli.go @@ -46,8 +46,10 @@ func Init(baseCmd *cobra.Command) { encryptCmd, decryptCmd, rolesCmd, + pinningCmd, ) rolesCmd.AddCommand(rolesGrantCmd, rolesLsCmd) + pinningCmd.AddCommand(pinningEnableCmd, pinningDisableCmd, pinningAddCmd, pinningRmCmd, pinningLsCmd) baseCmd.PersistentFlags().String("key", "", "Bucket key") baseCmd.PersistentFlags().String("thread", "", "Thread ID") @@ -79,6 +81,8 @@ func Init(baseCmd *cobra.Command) { rolesGrantCmd.Flags().StringP("role", "r", "", "Access role: none, reader, writer, admin") linksCmd.Flags().String("format", "default", "Display URL links in the provided format. Options: [default,json]") + + pinningAddCmd.Flags().String("name", "", "Optional pin name") } func SetBucks(b *local.Buckets) { diff --git a/cmd/buck/cli/pinning.go b/cmd/buck/cli/pinning.go new file mode 100644 index 0000000..dc330a6 --- /dev/null +++ b/cmd/buck/cli/pinning.go @@ -0,0 +1,163 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "runtime" + "time" + + "github.com/ipfs/go-cid" + "github.com/spf13/cobra" + "github.com/textileio/go-buckets/cmd" +) + +var pinningCmd = &cobra.Command{ + Use: "pinning", + Short: "Manage IPFS pinning service", + Long: `Enable/disable bucket as an IPFS Pinning Service.`, + Args: cobra.ExactArgs(0), +} + +var pinningEnableCmd = &cobra.Command{ + Use: "enable", + Short: "Enable bucket as an IPFS Pinning Service", + Long: `Enables the bucket as an IPFS Pinning Service for a locally running IPFS node.`, + Args: cobra.ExactArgs(0), + Run: func(c *cobra.Command, args []string) { + conf, err := bucks.NewConfigFromCmd(c, ".") + cmd.ErrCheck(err) + ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) + defer cancel() + buck, err := bucks.GetLocalBucket(ctx, conf) + cmd.ErrCheck(err) + + token, err := buck.GetIdentityToken(time.Hour * 24 * 30) + cmd.ErrCheck(err) + + links, err := buck.RemoteLinks(ctx, "") + cmd.ErrCheck(err) + + doExec( + fmt.Sprintf("ipfs pin remote service add %s %s %s", buck.Key(), links.BPS, string(token)), + nil) + + cmd.Success("Enabled bucket as pinning service: %s", aurora.White(buck.Key()).Bold()) + }, +} + +var pinningDisableCmd = &cobra.Command{ + Use: "disable", + Short: "Disable bucket as an IPFS Pinning Service", + Long: `Disables the bucket as an IPFS Pinning Service for a locally running IPFS node.`, + Args: cobra.ExactArgs(0), + Run: func(c *cobra.Command, args []string) { + conf, err := bucks.NewConfigFromCmd(c, ".") + cmd.ErrCheck(err) + ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) + defer cancel() + buck, err := bucks.GetLocalBucket(ctx, conf) + cmd.ErrCheck(err) + + doExec(fmt.Sprintf("ipfs pin remote service rm %s", buck.Key()), nil) + + cmd.Success("Disabled bucket as pinning service") + }, +} + +var pinningAddCmd = &cobra.Command{ + Use: "add [path]", + Short: "Add path to local IPFS node and pin to bucket", + Long: `Adds the path to the locally running IPFS node and pins the resulting CID to the bucket`, + Args: cobra.ExactArgs(1), + Run: func(c *cobra.Command, args []string) { + name, err := c.Flags().GetString("name") + cmd.ErrCheck(err) + + conf, err := bucks.NewConfigFromCmd(c, ".") + cmd.ErrCheck(err) + ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) + defer cancel() + buck, err := bucks.GetLocalBucket(ctx, conf) + cmd.ErrCheck(err) + + var buf bytes.Buffer + doExec(fmt.Sprintf("ipfs add -Qr --cid-version=1 %s", args[0]), &buf) + + doExec( + fmt.Sprintf("ipfs pin remote add --background --service=%s --name=%s %s", + buck.Key(), + name, + buf.String(), + ), + os.Stdout, + ) + }, +} + +var pinningRmCmd = &cobra.Command{ + Use: "rm [name|CID]", + Short: "Remove pin by name or CID from bucket", + Long: "Removes a pin by name or CID from the bucket.", + Args: cobra.ExactArgs(1), + Run: func(c *cobra.Command, args []string) { + conf, err := bucks.NewConfigFromCmd(c, ".") + cmd.ErrCheck(err) + ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) + defer cancel() + buck, err := bucks.GetLocalBucket(ctx, conf) + cmd.ErrCheck(err) + + var match string + if _, err := cid.Decode(args[0]); err == nil { + match = fmt.Sprintf("--cid=%s", args[0]) + } else { + match = fmt.Sprintf("--name=%s", args[0]) + } + + doExec( + fmt.Sprintf( + "ipfs pin remote rm --service=%s --status=queued,pinning,pinned,failed %s", + buck.Key(), + match, + ), + os.Stdout, + ) + }, +} + +var pinningLsCmd = &cobra.Command{ + Use: "ls", + Short: "Disable bucket as an IPFS Pinning Service", + Long: `Disables the bucket as an IPFS Pinning Service for a locally running IPFS node.`, + Args: cobra.ExactArgs(0), + Run: func(c *cobra.Command, args []string) { + conf, err := bucks.NewConfigFromCmd(c, ".") + cmd.ErrCheck(err) + ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) + defer cancel() + buck, err := bucks.GetLocalBucket(ctx, conf) + cmd.ErrCheck(err) + + doExec( + fmt.Sprintf("ipfs pin remote ls --service=%s --status=queued,pinning,pinned,failed", buck.Key()), + os.Stdout) + }, +} + +func doExec(c string, out io.Writer) { + var com *exec.Cmd + if runtime.GOOS == "windows" { + com = exec.Command("cmd", "/C", c) + } else { + com = exec.Command("bash", "-c", c) + } + if out != nil { + com.Stdout = out + } + err := com.Run() + cmd.ErrCheck(err) +} diff --git a/cmd/buck/cli/roles.go b/cmd/buck/cli/roles.go index 1725f04..e6c0238 100644 --- a/cmd/buck/cli/roles.go +++ b/cmd/buck/cli/roles.go @@ -40,7 +40,7 @@ Access roles: cmd.ErrCheck(err) conf, err := bucks.NewConfigFromCmd(c, ".") cmd.ErrCheck(err) - ctx, cancel := context.WithTimeout(context.Background(), cmd.PullTimeout) + ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) defer cancel() buck, err := bucks.GetLocalBucket(ctx, conf) cmd.ErrCheck(err) @@ -97,7 +97,7 @@ var rolesLsCmd = &cobra.Command{ Run: func(c *cobra.Command, args []string) { conf, err := bucks.NewConfigFromCmd(c, ".") cmd.ErrCheck(err) - ctx, cancel := context.WithTimeout(context.Background(), cmd.PullTimeout) + ctx, cancel := context.WithTimeout(context.Background(), cmd.Timeout) defer cancel() buck, err := bucks.GetLocalBucket(ctx, conf) cmd.ErrCheck(err) diff --git a/cmd/buckd/Dockerfile b/cmd/buckd/Dockerfile index 91508b9..bb19cd5 100644 --- a/cmd/buckd/Dockerfile +++ b/cmd/buckd/Dockerfile @@ -14,7 +14,7 @@ ENV SRC_DIR /go-buckets # Download packages first so they can be cached. COPY go.mod go.sum $SRC_DIR/ RUN cd $SRC_DIR \ - && go mod download + && CGO_ENABLED=0 go mod download COPY . $SRC_DIR diff --git a/cmd/buckd/Dockerfile.dev b/cmd/buckd/Dockerfile.dev index 1ee954c..2293867 100644 --- a/cmd/buckd/Dockerfile.dev +++ b/cmd/buckd/Dockerfile.dev @@ -8,11 +8,11 @@ ENV SRC_DIR /go-buckets COPY go.mod go.sum $SRC_DIR/ RUN cd $SRC_DIR \ - && go mod download + && CGO_ENABLED=0 go mod download COPY . $SRC_DIR -RUN cd $SRC_DIR \ +RUN --mount=type=cache,target=/root/.cache/go-build cd $SRC_DIR \ && CGO_ENABLED=0 GOOS=linux go build -gcflags "all=-N -l" -o buckd cmd/buckd/main.go FROM debian:buster diff --git a/cmd/buckd/docker-compose-dev.yml b/cmd/buckd/docker-compose-dev.yml index 6e9f218..63b9343 100644 --- a/cmd/buckd/docker-compose-dev.yml +++ b/cmd/buckd/docker-compose-dev.yml @@ -6,6 +6,7 @@ services: dockerfile: ./cmd/buckd/Dockerfile.dev volumes: - "${REPO_PATH}/buckets:/data/buckets" + platform: linux/amd64 environment: - BUCK_LOG_DEBUG=true - BUCK_ADDR_API=0.0.0.0:5000 @@ -16,6 +17,7 @@ services: - BUCK_GATEWAY_SUBDOMAINS - BUCK_GATEWAY_WWW_DOMAIN - BUCK_THREADS_ADDR=threads:5000 + - BUCK_THREADS_GATEWAY_URL=http://127.0.0.1:7000 - BUCK_IPFS_MULTIADDR=/dns4/ipfs/tcp/5001 - BUCK_IPNS_REPUBLISH_SCHEDULE - BUCK_IPNS_REPUBLISH_CONCURRENCY @@ -33,26 +35,32 @@ services: depends_on: - threads - ipfs + restart: unless-stopped threads: - image: textile/go-threads:534a6d0 - restart: always + image: textile/threads:sha-43a1673 volumes: - "${REPO_PATH}/threads:/data/threads" + platform: linux/amd64 environment: - THREADS_DEBUG=true - THREADS_APIADDR=/ip4/0.0.0.0/tcp/5000 - THREADS_APIPROXYADDR=/ip4/0.0.0.0/tcp/5050 + - THREADS_GATEWAYADDR=0.0.0.0:8000 ports: - "4066:4006" - "4066:4006/udp" - "127.0.0.1:4050:5050" + - "127.0.0.1:7000:8000" + restart: unless-stopped ipfs: - image: ipfs/go-ipfs:v0.8.0 + image: textile/go-ipfs:sha-ce693d7 volumes: - "${REPO_PATH}/ipfs:/data/ipfs" + platform: linux/amd64 environment: - IPFS_PROFILE=test ports: - "4011:4001" - "4011:4001/udp" - "127.0.0.1:8081:8080" + restart: unless-stopped diff --git a/cmd/buckd/docker-compose.yml b/cmd/buckd/docker-compose.yml index 0ea21fe..789f7cb 100644 --- a/cmd/buckd/docker-compose.yml +++ b/cmd/buckd/docker-compose.yml @@ -1,10 +1,10 @@ version: "3" services: buckets: - image: textile/textile:buckets - restart: always + image: textile/buckets:sha-7db2e12 volumes: - "${REPO_PATH}/buckets:/data/buckets" + platform: linux/amd64 environment: - BUCK_LOG_DEBUG - BUCK_ADDR_API=0.0.0.0:5000 @@ -17,6 +17,7 @@ services: - BUCK_GATEWAY_SUBDOMAINS - BUCK_GATEWAY_WWW_DOMAIN - BUCK_THREADS_ADDR=threads:5000 + - BUCK_THREADS_GATEWAY_URL=http://127.0.0.1:7000 - BUCK_IPFS_MULTIADDR=/dns4/ipfs/tcp/5001 - BUCK_IPNS_REPUBLISH_SCHEDULE - BUCK_IPNS_REPUBLISH_CONCURRENCY @@ -29,24 +30,29 @@ services: depends_on: - threads - ipfs + restart: unless-stopped threads: - image: textile/go-threads:534a6d0 - restart: always + image: textile/threads:sha-43a1673 volumes: - "${REPO_PATH}/threads:/data/threads" + platform: linux/amd64 environment: - THREADS_APIADDR=/ip4/0.0.0.0/tcp/5000 - THREADS_APIPROXYADDR=/ip4/0.0.0.0/tcp/5050 + - THREADS_GATEWAYADDR=0.0.0.0:8000 ports: - "4066:4006" - "4066:4006/udp" - "4050:5050" + - "7000:8000" + restart: unless-stopped ipfs: - image: ipfs/go-ipfs:v0.8.0 - restart: always + image: textile/go-ipfs:sha-ce693d7 volumes: - "${REPO_PATH}/ipfs:/data/ipfs" + platform: linux/amd64 ports: - "4011:4001" - "4011:4001/udp" - "8081:8080" + restart: unless-stopped diff --git a/cmd/buckd/main.go b/cmd/buckd/main.go index 86a6682..52896b9 100644 --- a/cmd/buckd/main.go +++ b/cmd/buckd/main.go @@ -7,11 +7,10 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" "time" - ds "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger" httpapi "github.com/ipfs/go-ipfs-http-client" logging "github.com/ipfs/go-log/v2" "github.com/spf13/cobra" @@ -22,9 +21,12 @@ import ( dns "github.com/textileio/go-buckets/dns" "github.com/textileio/go-buckets/gateway" ipns "github.com/textileio/go-buckets/ipns" + "github.com/textileio/go-buckets/pinning" + badger "github.com/textileio/go-ds-badger3" mongods "github.com/textileio/go-ds-mongo" dbc "github.com/textileio/go-threads/api/client" "github.com/textileio/go-threads/core/did" + kt "github.com/textileio/go-threads/db/keytransform" nc "github.com/textileio/go-threads/net/api/client" "github.com/textileio/go-threads/util" "google.golang.org/grpc" @@ -101,6 +103,10 @@ var ( Key: "threads.addr", DefValue: "127.0.0.1:4000", }, + "threadsGatewayUrl": { + Key: "threads.gateway_url", + DefValue: "http://127.0.0.1:7000", + }, // IPFS "ipfsMultiaddr": { @@ -204,6 +210,10 @@ func init() { "threadsAddr", config.Flags["threadsAddr"].DefValue.(string), "Threads API address") + rootCmd.PersistentFlags().String( + "threadsGatewayUrl", + config.Flags["threadsGatewayUrl"].DefValue.(string), + "Threads Gateway URL") // IPFS rootCmd.PersistentFlags().String( @@ -249,12 +259,14 @@ var rootCmd = &cobra.Command{ if config.Viper.GetBool("log.debug") { err := util.SetLogLevels(map[string]logging.LogLevel{ - daemonName: logging.LevelDebug, - "buckets": logging.LevelDebug, - "buckets-api": logging.LevelDebug, - "buckets-gateway": logging.LevelDebug, - "buckets-ipns": logging.LevelDebug, - "buckets-dns": logging.LevelDebug, + daemonName: logging.LevelDebug, + "buckets": logging.LevelDebug, + "buckets/api": logging.LevelDebug, + "buckets/ipns": logging.LevelDebug, + "buckets/dns": logging.LevelDebug, + "buckets/ps": logging.LevelDebug, + "buckets/ps-queue": logging.LevelDebug, + "buckets/gateway": logging.LevelDebug, }) cmd.ErrCheck(err) } @@ -284,6 +296,8 @@ var rootCmd = &cobra.Command{ gatewayWwwDomain := config.Viper.GetString("gateway.www_domain") threadsApi := config.Viper.GetString("threads.addr") + threadsGatewayUrl := config.Viper.GetString("threads.gateway_url") + ipfsApi := cmd.AddrFromStr(config.Viper.GetString("ipfs.multiaddr")) //ipnsRepublishSchedule := config.Viper.GetString("ipns.republish_schedule") @@ -299,16 +313,20 @@ var rootCmd = &cobra.Command{ ipfs, err := httpapi.NewApi(ipfsApi) cmd.ErrCheck(err) - var ipnsms ds.TxnDatastore + var ipnsms, pss kt.TxnDatastoreExtended switch datastoreType { case "badger": - ipnsms, err = newBadgerStore(datastoreBadgerRepo) + ipnsms, err = newBadgerStore(filepath.Join(datastoreBadgerRepo, "ipns")) + cmd.ErrCheck(err) + pss, err = newBadgerStore(filepath.Join(datastoreBadgerRepo, "pinq")) cmd.ErrCheck(err) case "mongo": ctx, cancel := context.WithCancel(context.Background()) defer cancel() ipnsms, err = newMongoStore(ctx, datastoreMongoUri, datastoreMongoName, "ipns") cmd.ErrCheck(err) + pss, err = newMongoStore(ctx, datastoreMongoUri, datastoreMongoName, "pinq") + cmd.ErrCheck(err) default: cmd.Fatal(errors.New("datastoreType must be 'badger' or 'mongo'")) } @@ -327,28 +345,37 @@ var rootCmd = &cobra.Command{ cmd.ErrCheck(err) buckets.GatewayURL = gatewayUrl + buckets.ThreadsGatewayURL = threadsGatewayUrl buckets.WWWDomain = gatewayWwwDomain server, proxy, err := common.GetServerAndProxy(lib, addrApi, addrApiProxy) cmd.ErrCheck(err) // Configure gateway - gateway, err := gateway.NewGateway(lib, ipfs, ipnsm, gateway.Config{ + ps, err := pinning.NewService(lib, pss) + cmd.ErrCheck(err) + gw, err := gateway.NewGateway(lib, ipfs, ipnsm, ps, gateway.Config{ Addr: addrGateway, URL: gatewayUrl, Domain: gatewayWwwDomain, Subdomains: gatewaySubdomains, }) cmd.ErrCheck(err) - gateway.Start() + gw.Start() fmt.Println("Welcome to Buckets!") cmd.HandleInterrupt(func() { - err := gateway.Close() + err := gw.Close() cmd.LogErr(err) log.Info("gateway was shutdown") + err = ps.Close() + cmd.LogErr(err) + err = pss.Close() + cmd.LogErr(err) + log.Info("pinning service was shutdown") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() err = proxy.Shutdown(ctx) @@ -375,8 +402,14 @@ var rootCmd = &cobra.Command{ err = ipnsm.Close() cmd.LogErr(err) + err = ipnsms.Close() + cmd.LogErr(err) log.Info("ipns manager was shutdown") + err = db.Close() + cmd.LogErr(err) + log.Info("db client was shutdown") + err = net.Close() cmd.LogErr(err) log.Info("net client was shutdown") @@ -397,13 +430,13 @@ func getClientRPCOpts(target string) (opts []grpc.DialOption) { return opts } -func newBadgerStore(repo string) (ds.TxnDatastore, error) { +func newBadgerStore(repo string) (kt.TxnDatastoreExtended, error) { if err := os.MkdirAll(repo, os.ModePerm); err != nil { return nil, err } return badger.NewDatastore(repo, &badger.DefaultOptions) } -func newMongoStore(ctx context.Context, uri, db, collection string) (ds.TxnDatastore, error) { +func newMongoStore(ctx context.Context, uri, db, collection string) (kt.TxnDatastoreExtended, error) { return mongods.New(ctx, uri, db, mongods.WithCollName(collection)) } diff --git a/collection/bucket.go b/collection/bucket.go index 79f4917..8fe21d6 100644 --- a/collection/bucket.go +++ b/collection/bucket.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "github.com/textileio/go-buckets/collection/mergemap" + "github.com/alecthomas/jsonschema" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/textileio/dcrypto" @@ -101,9 +103,10 @@ func (r Role) String() string { // Metadata contains metadata about a bucket item (a file or folder). type Metadata struct { - Key string `json:"key,omitempty"` - Roles map[did.DID]Role `json:"roles"` - UpdatedAt int64 `json:"updated_at"` + Key string `json:"key,omitempty"` + Roles map[did.DID]Role `json:"roles"` + Info map[string]interface{} `json:"info,omitempty"` + UpdatedAt int64 `json:"updated_at"` } // NewDefaultMetadata returns the default metadata for a path. @@ -254,12 +257,20 @@ func (b *Bucket) SetMetadataAtPath(pth string, md Metadata) { if md.Roles != nil { x.Roles = md.Roles } + if x.Info == nil { + x.Info = md.Info + } else if md.Info != nil { + mergemap.Merge(x.Info, md.Info) + } x.UpdatedAt = md.UpdatedAt b.Metadata[pth] = x } else { if md.Roles == nil { md.Roles = make(map[did.DID]Role) } + if md.Info == nil { + md.Info = make(map[string]interface{}) + } b.Metadata[pth] = md } } @@ -277,11 +288,38 @@ func (b *Bucket) UnsetMetadataWithPrefix(pre string) { } } -// ensureNoNulls inflates any values that are nil due to schema updates. -func (b *Bucket) ensureNoNulls() { - if b.Metadata == nil { - b.Metadata = make(map[string]Metadata) +// IsReadablePath returns whether or not a bucket path is readable by a did.DID. +func (b *Bucket) IsReadablePath(pth string, id did.DID) bool { + md, _, ok := b.GetMetadataForPath(pth, false) + if !ok { + return false + } + role, ok := md.Roles["*"] + if ok && role > NoneRole { + return true } + role, ok = md.Roles[id] + if ok && role > NoneRole { + return true + } + return false +} + +// IsWritablePath returns whether or not a bucket path is writable by a did.DID. +func (b *Bucket) IsWritablePath(pth string, id did.DID) bool { + md, _, ok := b.GetMetadataForPath(pth, false) + if !ok { + return false + } + role, ok := md.Roles["*"] + if ok && role > ReaderRole { + return true + } + role, ok = md.Roles[id] + if ok && role > ReaderRole { + return true + } + return false } // Copy returns a copy of the bucket. @@ -303,6 +341,13 @@ func (b *Bucket) Copy() *Bucket { } } +// ensureNoNulls inflates any values that are nil due to schema updates. +func (b *Bucket) ensureNoNulls() { + if b.Metadata == nil { + b.Metadata = make(map[string]Metadata) + } +} + // BucketOptions defines options for interacting with buckets. type BucketOptions struct { Name string @@ -484,7 +529,7 @@ func NewBuckets(c *dbc.Client) (*Buckets, error) { }, nil } -// Create a bucket instance. +// New creates a bucket instance. // Owner must be the identity token's subject. func (b *Buckets) New( ctx context.Context, diff --git a/collection/mergemap/LICENSE b/collection/mergemap/LICENSE new file mode 100644 index 0000000..22bf08c --- /dev/null +++ b/collection/mergemap/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2013, Peter Bourgon, SoundCloud Ltd. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/collection/mergemap/mergemap.go b/collection/mergemap/mergemap.go new file mode 100644 index 0000000..795ce47 --- /dev/null +++ b/collection/mergemap/mergemap.go @@ -0,0 +1,40 @@ +package mergemap + +import "reflect" + +var MaxDepth = 32 + +// Merge recursively merges the src and dst maps. Key conflicts are resolved by +// preferring src, or recursively descending, if both src and dst are maps. +func Merge(dst, src map[string]interface{}) map[string]interface{} { + return merge(dst, src, 0) +} + +func merge(dst, src map[string]interface{}, depth int) map[string]interface{} { + if depth > MaxDepth { + return dst + } + for key, srcVal := range src { + if dstVal, ok := dst[key]; ok { + srcMap, srcMapOk := mapify(srcVal) + dstMap, dstMapOk := mapify(dstVal) + if srcMapOk && dstMapOk { + srcVal = merge(dstMap, srcMap, depth+1) + } + } + dst[key] = srcVal + } + return dst +} + +func mapify(i interface{}) (map[string]interface{}, bool) { + value := reflect.ValueOf(i) + if value.Kind() == reflect.Map { + m := map[string]interface{}{} + for _, k := range value.MapKeys() { + m[k.String()] = value.MapIndex(k).Interface() + } + return m, true + } + return map[string]interface{}{}, false +} diff --git a/create.go b/create.go index bd16a62..5ce7c7b 100644 --- a/create.go +++ b/create.go @@ -14,6 +14,8 @@ import ( "github.com/textileio/go-threads/db" ) +// Create a new bucket using identity. +// See CreateOption for more details. func (b *Buckets) Create( ctx context.Context, identity did.Token, @@ -30,7 +32,12 @@ func (b *Buckets) Create( } } else { args.Thread = core.NewRandomIDV1() - if err := b.db.NewDB(ctx, args.Thread, db.WithNewManagedName(args.Name)); err != nil { + if err := b.db.NewDB( + ctx, + args.Thread, + db.WithNewManagedName(args.Name), + db.WithNewManagedToken(identity), + ); err != nil { return nil, nil, 0, fmt.Errorf("creating new thread: %v", err) } } diff --git a/dag/dag.go b/dag/dag.go index a1d8411..2bb1839 100644 --- a/dag/dag.go +++ b/dag/dag.go @@ -15,7 +15,6 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/textileio/go-buckets/collection" - "github.com/textileio/go-buckets/util" ) // MakeBucketSeed returns a raw ipld node containing a random seed. @@ -203,7 +202,7 @@ func GetNodeAtPath( key []byte, ) (ipld.Node, error) { if key != nil { - rp, fp, err := util.ParsePath(pth) + rp, fp, err := ParsePath(pth) if err != nil { return nil, err } @@ -300,7 +299,7 @@ func InsertNodeAtPath( key []byte, ) (context.Context, path.Resolved, error) { // The first step here is find a resolvable list of nodes that point to path. - rp, fp, err := util.ParsePath(pth) + rp, fp, err := ParsePath(pth) if err != nil { return ctx, nil, err } @@ -408,7 +407,7 @@ func RemoveNodeAtPath( key []byte, ) (context.Context, path.Resolved, error) { // The first step here is find a resolvable list of nodes that point to path. - rp, fp, err := util.ParsePath(pth) + rp, fp, err := ParsePath(pth) if err != nil { return ctx, nil, err } @@ -482,3 +481,33 @@ func GetPathSize(ctx context.Context, ipfs iface.CoreAPI, root path.Path) (int64 } return int64(stat.CumulativeSize), nil } + +// NewResolvedPath returns path.Resolved from a string. +func NewResolvedPath(s string) (path.Resolved, error) { + parts := strings.SplitN(s, "/", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("path is not resolvable") + } + c, err := cid.Decode(parts[2]) + if err != nil { + return nil, err + } + return path.IpfsPath(c), nil +} + +// ParsePath returns path.Resolved and a path remainder from path.Path. +func ParsePath(p path.Path) (resolved path.Resolved, fpath string, err error) { + parts := strings.SplitN(p.String(), "/", 4) + if len(parts) < 3 { + err = fmt.Errorf("path does not contain a resolvable segment") + return + } + c, err := cid.Decode(parts[2]) + if err != nil { + return + } + if len(parts) > 3 { + fpath = parts[3] + } + return path.IpfsPath(c), fpath, nil +} diff --git a/dns/dns.go b/dns/dns.go index f251019..cf50bdf 100644 --- a/dns/dns.go +++ b/dns/dns.go @@ -7,7 +7,7 @@ import ( logging "github.com/ipfs/go-log/v2" ) -var log = logging.Logger("buckets-dns") +var log = logging.Logger("buckets/dns") const IPFSGateway = "cloudflare-ipfs.com" @@ -94,7 +94,7 @@ func (m *Manager) UpdateRecord(id, rtype, name, content string) error { return nil } -// Delete removes a record by ID from dns. +// DeleteRecord removes a record by ID from dns. func (m *Manager) DeleteRecord(id string) error { if err := m.api.DeleteDNSRecord(m.zoneID, id); err != nil { return err diff --git a/gateway/bucketfs.go b/gateway/bucketfs.go new file mode 100644 index 0000000..d388b7c --- /dev/null +++ b/gateway/bucketfs.go @@ -0,0 +1,168 @@ +package gateway + +import ( + "context" + "errors" + "fmt" + "net/http" + gopath "path" + "strings" + + "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/render" + "github.com/textileio/go-assets" + "github.com/textileio/go-buckets" + "github.com/textileio/go-buckets/ipns" + "github.com/textileio/go-threads/core/did" + core "github.com/textileio/go-threads/core/thread" +) + +type fileSystem struct { + *assets.FileSystem +} + +func (f *fileSystem) Exists(prefix, path string) bool { + pth := strings.TrimPrefix(path, prefix) + if pth == "/" { + return false + } + _, ok := f.Files[pth] + return ok +} + +type serveBucketFS interface { + GetThread(key string) (core.ID, error) + Exists(ctx context.Context, thread core.ID, bucket, pth string, token did.Token) (bool, string) + Write(c *gin.Context, ctx context.Context, thread core.ID, bucket, pth string, token did.Token) error + ValidHost() string +} + +type bucketFS struct { + lib *buckets.Buckets + ipns *ipns.Manager + domain string +} + +func serveBucket(fs serveBucketFS) gin.HandlerFunc { + return func(c *gin.Context) { + key, err := bucketFromHost(c.Request.Host, fs.ValidHost()) + if err != nil { + return + } + thread, err := fs.GetThread(key) + if err != nil { + return + } + token := did.Token(c.Query("token")) + + var content string + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + exists, target := fs.Exists(ctx, thread, key, c.Request.URL.Path, token) + if exists { + content = c.Request.URL.Path + } else if len(target) != 0 { + content = gopath.Join(c.Request.URL.Path, target) + } + if len(content) != 0 { + if err := fs.Write(c, ctx, thread, key, content, token); err != nil { + renderError(c, http.StatusInternalServerError, err) + } else { + c.Abort() + } + } + } +} + +func (f *bucketFS) GetThread(bkey string) (id core.ID, err error) { + key, err := f.ipns.Store().GetByCid(bkey) + if err != nil { + return + } + return key.ThreadID, nil +} + +func (f *bucketFS) Exists(ctx context.Context, thread core.ID, key, pth string, token did.Token) (ok bool, name string) { + if key == "" || pth == "/" { + return + } + rep, _, err := f.lib.ListPath(ctx, thread, key, token, pth) + if err != nil { + return + } + if rep.IsDir { + for _, item := range rep.Items { + if item.Name == "index.html" { + return false, item.Name + } + } + return + } + return true, "" +} + +func (f *bucketFS) Write(c *gin.Context, ctx context.Context, thread core.ID, key, pth string, token did.Token) error { + r, err := f.lib.PullPath(ctx, thread, key, token, pth) + if err != nil { + return fmt.Errorf("pulling path: %v", err) + } + defer r.Close() + + ct, mr, err := detectReaderOrPathContentType(r, pth) + if err != nil { + return fmt.Errorf("detecting content-type: %v", err) + } + c.Writer.Header().Set("Content-Type", ct) + c.Render(200, render.Reader{ContentLength: -1, Reader: mr}) + return nil +} + +func (f *bucketFS) ValidHost() string { + return f.domain +} + +func (g *Gateway) renderWWWBucket(c *gin.Context, key string) { + ipnskey, err := g.ipns.Store().GetByCid(key) + if err != nil { + render404(c) + return + } + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + token := did.Token(c.Query("token")) + rep, _, err := g.lib.ListPath(ctx, ipnskey.ThreadID, key, token, "") + if err != nil { + render404(c) + return + } + for _, item := range rep.Items { + if item.Name == "index.html" { + r, err := g.lib.PullPath(ctx, ipnskey.ThreadID, key, token, item.Name) + if err != nil { + render404(c) + return + } + + ct, mr, err := detectReaderOrPathContentType(r, item.Name) + if err != nil { + renderError(c, http.StatusInternalServerError, fmt.Errorf("detecting content-type: %v", err)) + return + } + c.Writer.Header().Set("Content-Type", ct) + c.Render(200, render.Reader{ContentLength: -1, Reader: mr}) + r.Close() + } + } + renderError(c, http.StatusNotFound, errors.New("an index.html file was not found in this bucket")) +} + +func bucketFromHost(host, valid string) (key string, err error) { + parts := strings.SplitN(host, ".", 2) + hostport := parts[len(parts)-1] + hostparts := strings.SplitN(hostport, ":", 2) + if hostparts[0] != valid || valid == "" { + err = errors.New("invalid bucket host") + return + } + return parts[0], nil +} diff --git a/gateway/buckets.go b/gateway/buckets.go index dccdfb8..b566c1a 100644 --- a/gateway/buckets.go +++ b/gateway/buckets.go @@ -3,125 +3,82 @@ package gateway import ( "context" "fmt" - "io" - "mime" "net/http" - "path" - "path/filepath" + gopath "path" "strconv" "strings" "time" "github.com/gin-gonic/gin" - assets "github.com/textileio/go-assets" - "github.com/textileio/go-buckets" + "github.com/gin-gonic/gin/render" "github.com/textileio/go-buckets/collection" - "github.com/textileio/go-buckets/ipns" - "github.com/textileio/go-buckets/util" "github.com/textileio/go-threads/core/did" - "github.com/textileio/go-threads/core/thread" + core "github.com/textileio/go-threads/core/thread" ) -type fileSystem struct { - *assets.FileSystem -} - -func (f *fileSystem) Exists(prefix, path string) bool { - pth := strings.TrimPrefix(path, prefix) - if pth == "/" { - return false - } - _, ok := f.Files[pth] - return ok -} - -func (g *Gateway) renderBucket(c *gin.Context, ctx context.Context, threadID thread.ID, token did.Token) { - rep, err := g.lib.List(ctx, threadID, token) +func (g *Gateway) bucketHandler(c *gin.Context) { + thread, err := g.getThread(c) if err != nil { renderError(c, http.StatusBadRequest, err) return } - links := make([]link, len(rep)) - for i, r := range rep { - var name string - if r.Name != "" { - name = r.Name - } else { - name = r.Key - } - p := path.Join("thread", threadID.String(), collection.Name, r.Key) - if token.Defined() { - p += "?token=" + string(token) - } - links[i] = link{ - Name: name, - Path: p, - Size: "", - Links: "", - } - } - c.HTML(http.StatusOK, "/public/html/unixfs.gohtml", gin.H{ - "Title": "Index of " + path.Join("/thread", threadID.String(), collection.Name), - "Root": "/", - "Path": "", - "Updated": "", - "Back": "", - "Links": links, - }) + g.renderBucket(c, thread, c.Param("key"), c.Param("path")) } -func (g *Gateway) renderBucketPath( - c *gin.Context, - ctx context.Context, - threadID thread.ID, - id, - pth string, - token did.Token, -) { - rep, buck, err := g.lib.ListPath(ctx, threadID, id, pth, token) +func (g *Gateway) renderBucket(c *gin.Context, thread core.ID, key, pth string) { + pth = strings.TrimPrefix(pth, "/") + token := did.Token(c.Query("token")) + + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + rep, buck, err := g.lib.ListPath(ctx, thread, key, token, pth) if err != nil { render404(c) return } if !rep.IsDir { - r, err := g.lib.PullPath(ctx, threadID, buck.Key, pth, token) + r, err := g.lib.PullPath(ctx, thread, buck.Key, token, pth) if err != nil { render404(c) return } defer r.Close() - if _, err := io.Copy(c.Writer, r); err != nil { - render404(c) + + ct, mr, err := detectReaderOrPathContentType(r, pth) + if err != nil { + renderError(c, http.StatusInternalServerError, fmt.Errorf("detecting content-type: %v", err)) return } + c.Writer.Header().Set("Content-Type", ct) + c.Render(200, render.Reader{ContentLength: -1, Reader: mr}) } else { var base string if g.subdomains { base = collection.Name } else { - base = path.Join("thread", threadID.String(), collection.Name) + base = gopath.Join("thread", thread.String(), collection.Name) } var links []link for _, item := range rep.Items { - pth := path.Join(base, strings.Replace(item.Path, buck.Path, buck.Key, 1)) + pth := gopath.Join(base, strings.Replace(item.Path, buck.Path, buck.Key, 1)) if token.Defined() { pth += "?token=" + string(token) } links = append(links, link{ Name: item.Name, Path: pth, - Size: util.ByteCountDecimal(item.Size), + Size: byteCountDecimal(item.Size), Links: strconv.Itoa(len(item.Items)), }) } var name string - if buck.Name != "" { + if len(buck.Name) != 0 { name = buck.Name } else { name = buck.Key } root := strings.Replace(rep.Path, buck.Path, name, 1) - back := path.Dir(path.Join(base, strings.Replace(rep.Path, buck.Path, buck.Key, 1))) + back := gopath.Dir(gopath.Join(base, strings.Replace(rep.Path, buck.Path, buck.Key, 1))) if token.Defined() { back += "?token=" + string(token) } @@ -135,144 +92,3 @@ func (g *Gateway) renderBucketPath( }) } } - -type serveBucketFS interface { - GetThread(key string) (thread.ID, error) - Exists(ctx context.Context, threadID thread.ID, bucket, pth string, token did.Token) (bool, string) - Write(ctx context.Context, threadID thread.ID, bucket, pth string, token did.Token, writer io.Writer) error - ValidHost() string -} - -type bucketFS struct { - lib *buckets.Buckets - ipns *ipns.Manager - domain string -} - -func serveBucket(fs serveBucketFS) gin.HandlerFunc { - return func(c *gin.Context) { - key, err := bucketFromHost(c.Request.Host, fs.ValidHost()) - if err != nil { - return - } - threadID, err := fs.GetThread(key) - if err != nil { - return - } - token := did.Token(c.Query("token")) - - ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) - defer cancel() - exists, target := fs.Exists(ctx, threadID, key, c.Request.URL.Path, token) - if exists { - c.Writer.WriteHeader(http.StatusOK) - ctype := mime.TypeByExtension(filepath.Ext(c.Request.URL.Path)) - if ctype == "" { - ctype = "application/octet-stream" - } - c.Writer.Header().Set("Content-Type", ctype) - if err := fs.Write(ctx, threadID, key, c.Request.URL.Path, token, c.Writer); err != nil { - renderError(c, http.StatusInternalServerError, err) - } else { - c.Abort() - } - } else if target != "" { - content := path.Join(c.Request.URL.Path, target) - ctype := mime.TypeByExtension(filepath.Ext(content)) - c.Writer.WriteHeader(http.StatusOK) - c.Writer.Header().Set("Content-Type", ctype) - if err := fs.Write(ctx, threadID, key, content, token, c.Writer); err != nil { - renderError(c, http.StatusInternalServerError, err) - } else { - c.Abort() - } - } - } -} - -func (f *bucketFS) GetThread(bkey string) (id thread.ID, err error) { - key, err := f.ipns.Store().GetByCid(bkey) - if err != nil { - return - } - return key.ThreadID, nil -} - -func (f *bucketFS) Exists(ctx context.Context, threadID thread.ID, key, pth string, token did.Token) (ok bool, name string) { - if key == "" || pth == "/" { - return - } - rep, _, err := f.lib.ListPath(ctx, threadID, key, pth, token) - if err != nil { - return - } - if rep.IsDir { - for _, item := range rep.Items { - if item.Name == "index.html" { - return false, item.Name - } - } - return - } - return true, "" -} - -func (f *bucketFS) Write(ctx context.Context, threadID thread.ID, key, pth string, token did.Token, writer io.Writer) error { - r, err := f.lib.PullPath(ctx, threadID, key, pth, token) - if err != nil { - return err - } - defer r.Close() - _, err = io.Copy(writer, r) - return err -} - -func (f *bucketFS) ValidHost() string { - return f.domain -} - -// renderWWWBucket renders a bucket as a website. -func (g *Gateway) renderWWWBucket(c *gin.Context, key string) { - ipnskey, err := g.ipns.Store().GetByCid(key) - if err != nil { - render404(c) - return - } - ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) - defer cancel() - token := did.Token(c.Query("token")) - rep, _, err := g.lib.ListPath(ctx, ipnskey.ThreadID, key, "", token) - if err != nil { - render404(c) - return - } - for _, item := range rep.Items { - if item.Name == "index.html" { - c.Writer.WriteHeader(http.StatusOK) - c.Writer.Header().Set("Content-Type", "text/html") - r, err := g.lib.PullPath(ctx, ipnskey.ThreadID, key, item.Name, token) - if err != nil { - render404(c) - return - } - if _, err := io.Copy(c.Writer, r); err != nil { - r.Close() - render404(c) - return - } - r.Close() - } - } - renderError(c, http.StatusNotFound, fmt.Errorf("an index.html file was not found in this bucket")) -} - -func bucketFromHost(host, valid string) (key string, err error) { - parts := strings.SplitN(host, ".", 2) - hostport := parts[len(parts)-1] - hostparts := strings.SplitN(hostport, ":", 2) - if hostparts[0] != valid || valid == "" { - err = fmt.Errorf("invalid bucket host") - return - } - return parts[0], nil -} diff --git a/gateway/gateway.go b/gateway/gateway.go index 71ce6b0..34fe623 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -1,12 +1,18 @@ package gateway +// @todo: Migrate away from the current asset generator + import ( + "bytes" "context" + "errors" "fmt" "html/template" + "io" "io/ioutil" "net/http" "net/url" + "path/filepath" "strings" "time" @@ -23,10 +29,12 @@ import ( gincors "github.com/rs/cors/wrapper/gin" "github.com/textileio/go-buckets" "github.com/textileio/go-buckets/ipns" - "github.com/textileio/go-threads/core/thread" + "github.com/textileio/go-buckets/pinning" + "github.com/textileio/go-threads/core/did" + core "github.com/textileio/go-threads/core/thread" ) -var log = logging.Logger("buckets-gateway") +var log = logging.Logger("buckets/gateway") const handlerTimeout = time.Minute @@ -48,6 +56,7 @@ type Gateway struct { lib *buckets.Buckets ipfs iface.CoreAPI ipns *ipns.Manager + ps *pinning.Service addr string url string @@ -64,11 +73,18 @@ type Config struct { } // NewGateway returns a new gateway. -func NewGateway(lib *buckets.Buckets, ipfs iface.CoreAPI, ipns *ipns.Manager, conf Config) (*Gateway, error) { +func NewGateway( + lib *buckets.Buckets, + ipfs iface.CoreAPI, + ipns *ipns.Manager, + ps *pinning.Service, + conf Config, +) (*Gateway, error) { return &Gateway{ lib: lib, ipfs: ipfs, ipns: ipns, + ps: ps, addr: conf.Addr, url: conf.URL, domain: conf.Domain, @@ -99,18 +115,39 @@ func (g *Gateway) Start() { c.Writer.WriteHeader(http.StatusNoContent) }) - router.GET("/thread/:thread/:collection", g.subdomainOptionHandler, g.collectionHandler) - router.GET("/thread/:thread/:collection/:id", g.subdomainOptionHandler, g.instanceHandler) - router.GET("/thread/:thread/:collection/:id/*path", g.subdomainOptionHandler, g.instanceHandler) - + // IPFS router.GET("/ipfs/:root", g.subdomainOptionHandler, g.ipfsHandler) router.GET("/ipfs/:root/*path", g.subdomainOptionHandler, g.ipfsHandler) + + // IPNS router.GET("/ipns/:key", g.subdomainOptionHandler, g.ipnsHandler) router.GET("/ipns/:key/*path", g.subdomainOptionHandler, g.ipnsHandler) + + // P2P router.GET("/p2p/:key", g.subdomainOptionHandler, g.p2pHandler) + + // IPLD router.GET("/ipld/:root", g.subdomainOptionHandler, g.ipldHandler) router.GET("/ipld/:root/*path", g.subdomainOptionHandler, g.ipldHandler) + // Buckets + router.GET("/thread/:thread/buckets", g.subdomainOptionHandler, g.threadHandler) + router.POST("/thread/:thread/buckets/:key", g.subdomainOptionHandler, g.bucketPushPathsHandler) + router.GET("/thread/:thread/buckets/:key", g.subdomainOptionHandler, g.bucketHandler) + router.GET("/thread/:thread/buckets/:key/*path", g.subdomainOptionHandler, g.bucketHandler) + + // Buckets shorthand + router.POST("/b/:key", g.bucketPushPathsHandler) + router.GET("/b/:key", g.bucketHandler) + router.GET("/b/:key/*path", g.bucketHandler) + + // Buckets Pinning Service + router.GET("/bps/:key/pins", g.subdomainOptionHandler, g.listPinsHandler) + router.POST("/bps/:key/pins", g.subdomainOptionHandler, g.addPinHandler) + router.GET("/bps/:key/pins/:requestid", g.subdomainOptionHandler, g.getPinHandler) + router.POST("/bps/:key/pins/:requestid", g.subdomainOptionHandler, g.replacePinHandler) + router.DELETE("/bps/:key/pins/:requestid", g.subdomainOptionHandler, g.removePinHandler) + router.NoRoute(g.subdomainHandler) g.server = &http.Server{ @@ -125,58 +162,33 @@ func (g *Gateway) Start() { log.Infof("gateway listening at %s", g.server.Addr) } -// loadTemplate loads HTML templates. -func loadTemplate() (*template.Template, error) { - t := template.New("") - for name, file := range Assets.Files { - if file.IsDir() || !strings.HasSuffix(name, ".gohtml") { - continue - } - h, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - t, err = t.New(name).Parse(string(h)) - if err != nil { - return nil, err - } - } - return t, nil -} - -// Addr returns the gateway's address. -func (g *Gateway) Addr() string { - return g.server.Addr -} - // Close the gateway. func (g *Gateway) Close() error { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() if err := g.server.Shutdown(ctx); err != nil { - return err + return fmt.Errorf("shutting down server: %v", err) } return nil } -// subdomainOptionHandler redirects valid namespaces to subdomains if the option is enabled. -func (g *Gateway) subdomainOptionHandler(c *gin.Context) { - if !g.subdomains { - return - } - loc, ok := g.toSubdomainURL(c.Request) - if !ok { - render404(c) - return - } +// Addr returns the gateway's listen address. +func (g *Gateway) Addr() string { + return g.server.Addr +} - // See security note https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L105 - c.Request.Header.Set("Clear-Site-Data", "\"cookies\", \"storage\"") +// Url returns the gateway's externally configured URL. +func (g *Gateway) Url() string { + return g.url +} - c.Redirect(http.StatusPermanentRedirect, loc) +// Buckets returns the gateway's bucket lib. +func (g *Gateway) Buckets() *buckets.Buckets { + return g.lib } // dashboardHandler renders a dev or org dashboard. +// @todo: Use or remove dashboard handler func (g *Gateway) dashboardHandler(c *gin.Context) { render404(c) } @@ -201,6 +213,51 @@ func formatError(err error) string { return strings.Join(words, " ") + "." } +// getThread returns core.ID from request params or the IPNS key store. +func (g *Gateway) getThread(c *gin.Context) (core.ID, error) { + threadp := c.Param("thread") + if len(threadp) != 0 { + thread, err := core.Decode(threadp) + if err != nil { + return "", errors.New("invalid thread ID") + } + return thread, nil + } + // @todo: Use a fixed length in-mem cache of thread IDs + ipnskey, err := g.ipns.Store().GetByCid(c.Param("key")) + if err != nil { + return "", fmt.Errorf("looking up thread: %v", err) + } + return ipnskey.ThreadID, nil +} + +// getAuth returns did.Token from the authorization header. +func getAuth(c *gin.Context) (did.Token, bool) { + auth := strings.Split(c.Request.Header.Get("Authorization"), " ") + if len(auth) < 2 { + return "", false + } + return did.Token(auth[1]), true +} + +// subdomainOptionHandler redirects valid namespaces to subdomains if the option is enabled. +func (g *Gateway) subdomainOptionHandler(c *gin.Context) { + if !g.subdomains { + return + } + loc, ok := g.toSubdomainURL(c.Request) + if !ok { + render404(c) + return + } + + // See security note: + // https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L105 + c.Request.Header.Set("Clear-Site-Data", "\"cookies\", \"storage\"") + + c.Redirect(http.StatusPermanentRedirect, loc) +} + // subdomainHandler handles requests by parsing the request subdomain. func (g *Gateway) subdomainHandler(c *gin.Context) { c.Status(200) @@ -233,26 +290,51 @@ func (g *Gateway) subdomainHandler(c *gin.Context) { case "ipld": g.renderIPLDPath(c, key+c.Request.URL.Path) case "thread": - threadID, err := thread.Decode(key) + thread, err := core.Decode(key) if err != nil { - renderError(c, http.StatusBadRequest, fmt.Errorf("invalid thread ID")) + renderError(c, http.StatusBadRequest, errors.New("invalid thread ID")) return } parts := strings.SplitN(strings.TrimSuffix(c.Request.URL.Path, "/"), "/", 4) + if len(parts) < 2 || parts[1] != "buckets" { + render404(c) + return + } switch len(parts) { - case 1: - // @todo: Render something at the thread root + case 2: + g.renderThread(c, thread) + case 3: + if c.Request.Method == "POST" { + g.pushBucketPaths(c, thread, parts[2]) + } else { + g.renderBucket(c, thread, parts[2], "") + } + case 4: + g.renderBucket(c, thread, parts[2], parts[3]) + default: render404(c) + } + case "bps": + parts := strings.SplitN(strings.TrimSuffix(c.Request.URL.Path, "/"), "/", 3) + if len(parts) < 2 || parts[1] != "pins" { + render404(c) + return + } + switch len(parts) { case 2: - if parts[1] != "" { - g.renderCollection(c, threadID, parts[1]) + if c.Request.Method == "POST" { + g.addPin(c, key) } else { - render404(c) + g.listPins(c, key) } case 3: - g.renderInstance(c, threadID, parts[1], parts[2], "") - case 4: - g.renderInstance(c, threadID, parts[1], parts[2], parts[3]) + if c.Request.Method == "POST" { + g.replacePin(c, key, parts[2]) + } else if c.Request.Method == "DELETE" { + g.removePin(c, key, parts[2]) + } else { + g.getPin(c, key, parts[2]) + } default: render404(c) } @@ -261,17 +343,19 @@ func (g *Gateway) subdomainHandler(c *gin.Context) { } } -// Modified from https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L251 +// Modified from: +// https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L251 func isSubdomainNamespace(ns string) bool { switch ns { - case "ipfs", "ipns", "p2p", "ipld", "thread": + case "ipfs", "ipns", "p2p", "ipld", "thread", "bps": return true default: return false } } -// Copied from https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L260 +// Copied from: +// https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L260 func isPeerIDNamespace(ns string) bool { switch ns { case "ipns", "p2p": @@ -282,7 +366,8 @@ func isPeerIDNamespace(ns string) bool { } // Converts a hostname/path to a subdomain-based URL, if applicable. -// Modified from https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L270 +// Modified from: +// https://github.com/ipfs/go-ipfs/blob/dbfa7bf2b216bad9bec1ff66b1f3814f4faac31e/core/corehttp/hostname.go#L270 func (g *Gateway) toSubdomainURL(r *http.Request) (redirURL string, ok bool) { var ns, rootID, rest string @@ -357,3 +442,60 @@ func (g *Gateway) toSubdomainURL(r *http.Request) (redirURL string, ok bool) { host := urlparts[1] return safeRedirectURL(fmt.Sprintf("%s://%s.%s.%s/%s%s", scheme, rootID, ns, host, rest, query)) } + +// detectReaderOrPathContentType detects the best available mime type for a reader, +// considering file extensions. http.DetectContentType does not properly detect content-type +// for web assets: htm, html, css, and js. +func detectReaderOrPathContentType(r io.Reader, pth string) (string, io.Reader, error) { + var buf [512]byte + n, err := io.ReadAtLeast(r, buf[:], len(buf)) + if err != nil && err != io.ErrUnexpectedEOF { + return "", nil, fmt.Errorf("reading reader: %s", err) + } + reader := io.MultiReader(bytes.NewReader(buf[:n]), r) + switch filepath.Ext(pth) { + case ".htm", ".html": + return "text/html", reader, nil + case ".css": + return "text/css", reader, nil + case ".js": + return "application/javascript", reader, nil + case ".txt", ".md": + return "text/plain", reader, nil + default: + return http.DetectContentType(buf[:]), reader, nil + } +} + +// byteCountDecimal returns a human readable byte size. +func byteCountDecimal(b int64) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) +} + +// loadTemplate loads HTML templates. +func loadTemplate() (*template.Template, error) { + t := template.New("") + for name, file := range Assets.Files { + if file.IsDir() || !strings.HasSuffix(name, ".gohtml") { + continue + } + h, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + t, err = t.New(name).Parse(string(h)) + if err != nil { + return nil, err + } + } + return t, nil +} diff --git a/gateway/ipfs.go b/gateway/ipfs.go index 6d8c84e..157903a 100644 --- a/gateway/ipfs.go +++ b/gateway/ipfs.go @@ -3,7 +3,7 @@ package gateway import ( "context" "fmt" - "io/ioutil" + "io" "net/http" gopath "path" "strings" @@ -14,7 +14,6 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-core/peer" - "github.com/textileio/go-buckets/util" ) func (g *Gateway) ipfsHandler(c *gin.Context) { @@ -27,7 +26,7 @@ func (g *Gateway) renderIPFSPath(c *gin.Context, base, pth string) { ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) defer cancel() pth = strings.TrimSuffix(pth, "/") - data, err := g.openPath(ctx, path.New(pth)) + f, err := g.openPath(ctx, path.New(pth)) if err != nil { if err == iface.ErrIsDir { var root, dir, back string @@ -58,7 +57,7 @@ func (g *Gateway) renderIPFSPath(c *gin.Context, base, pth string) { links = append(links, link{ Name: l.Name, Path: gopath.Join(dir, l.Name), - Size: util.ByteCountDecimal(int64(l.Size)), + Size: byteCountDecimal(int64(l.Size)), }) } var index string @@ -79,21 +78,29 @@ func (g *Gateway) renderIPFSPath(c *gin.Context, base, pth string) { "Links": links, } c.HTML(http.StatusOK, "/public/html/unixfs.gohtml", params) - } else { - renderError(c, http.StatusBadRequest, err) return } - } else { - c.Render(200, render.Data{Data: data}) + + renderError(c, http.StatusBadRequest, err) + return + } + defer f.Close() + + ct, r, err := detectReaderOrPathContentType(f, pth) + if err != nil { + renderError(c, http.StatusInternalServerError, fmt.Errorf("detecting mime: %s", err)) + return } + + c.Writer.Header().Set("Content-Type", ct) + c.Render(200, render.Reader{ContentLength: -1, Reader: r}) } -func (g *Gateway) openPath(ctx context.Context, pth path.Path) ([]byte, error) { +func (g *Gateway) openPath(ctx context.Context, pth path.Path) (io.ReadCloser, error) { f, err := g.ipfs.Unixfs().Get(ctx, pth) if err != nil { return nil, err } - defer f.Close() var file files.File switch f := f.(type) { case files.File: @@ -103,7 +110,7 @@ func (g *Gateway) openPath(ctx context.Context, pth path.Path) ([]byte, error) { default: return nil, iface.ErrNotSupported } - return ioutil.ReadAll(file) + return file, nil } func (g *Gateway) ipnsHandler(c *gin.Context) { @@ -111,6 +118,7 @@ func (g *Gateway) ipnsHandler(c *gin.Context) { } func (g *Gateway) renderIPNSKey(c *gin.Context, key, pth string) { + // @todo: Lookup key and render from local content if exists ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) defer cancel() root, err := g.ipfs.Name().Resolve(ctx, key) diff --git a/gateway/pinning.go b/gateway/pinning.go new file mode 100644 index 0000000..4ca38c2 --- /dev/null +++ b/gateway/pinning.go @@ -0,0 +1,265 @@ +package gateway + +import ( + "context" + "errors" + "net/http" + "regexp" + "strings" + + "github.com/gin-gonic/gin" + "github.com/textileio/go-buckets/pinning" + openapi "github.com/textileio/go-buckets/pinning/openapi/go" + "github.com/textileio/go-buckets/pinning/queue" +) + +var queryMapRx *regexp.Regexp + +func init() { + queryMapRx = regexp.MustCompile(`\[(.*?)]`) +} + +func (g *Gateway) listPinsHandler(c *gin.Context) { + g.listPins(c, c.Param("key")) +} + +func (g *Gateway) listPins(c *gin.Context, key string) { + thread, err := g.getThread(c) + if err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + token, ok := getAuth(c) + if !ok { + newFailure(c, http.StatusUnauthorized, errors.New("authorization required")) + return + } + + var query openapi.Query + if err := c.ShouldBindQuery(&query); err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + sq := getQuery(query) + + if m, ok := c.GetQuery("meta"); ok { + sq.Meta = getQueryMap(m) + } + + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + pins, err := g.ps.ListPins(ctx, thread, key, token, sq) + if err != nil { + handleServiceErr(c, err) + return + } + + res := openapi.PinResults{ + Count: int32(len(pins)), + Results: pins, + } + c.JSON(http.StatusOK, res) +} + +func (g *Gateway) addPinHandler(c *gin.Context) { + g.addPin(c, c.Param("key")) +} + +func (g *Gateway) addPin(c *gin.Context, key string) { + thread, err := g.getThread(c) + if err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + token, ok := getAuth(c) + if !ok { + newFailure(c, http.StatusUnauthorized, errors.New("authorization required")) + return + } + + var pin openapi.Pin + if err := c.ShouldBind(&pin); err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + status, err := g.ps.AddPin(ctx, thread, key, token, pin) + if err != nil { + handleServiceErr(c, err) + return + } + + c.JSON(http.StatusAccepted, status) +} + +func (g *Gateway) getPinHandler(c *gin.Context) { + g.getPin(c, c.Param("key"), c.Param("requestid")) +} + +func (g *Gateway) getPin(c *gin.Context, key, id string) { + thread, err := g.getThread(c) + if err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + token, ok := getAuth(c) + if !ok { + newFailure(c, http.StatusUnauthorized, errors.New("authorization required")) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + status, err := g.ps.GetPin(ctx, thread, key, token, id) + if err != nil { + handleServiceErr(c, err) + return + } + + c.JSON(http.StatusOK, status) +} + +func (g *Gateway) replacePinHandler(c *gin.Context) { + g.replacePin(c, c.Param("key"), c.Param("requestid")) +} + +func (g *Gateway) replacePin(c *gin.Context, key, id string) { + thread, err := g.getThread(c) + if err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + token, ok := getAuth(c) + if !ok { + newFailure(c, http.StatusUnauthorized, errors.New("authorization required")) + return + } + + var pin openapi.Pin + if err := c.ShouldBind(&pin); err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + status, err := g.ps.ReplacePin(ctx, thread, key, token, id, pin) + if err != nil { + handleServiceErr(c, err) + return + } + + c.JSON(http.StatusAccepted, status) +} + +func (g *Gateway) removePinHandler(c *gin.Context) { + g.removePin(c, c.Param("key"), c.Param("requestid")) +} + +func (g *Gateway) removePin(c *gin.Context, key, id string) { + thread, err := g.getThread(c) + if err != nil { + newFailure(c, http.StatusBadRequest, err) + return + } + token, ok := getAuth(c) + if !ok { + newFailure(c, http.StatusUnauthorized, errors.New("authorization required")) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + if err := g.ps.RemovePin(ctx, thread, key, token, id); err != nil { + handleServiceErr(c, err) + return + } + + c.Status(http.StatusAccepted) +} + +func getQuery(q openapi.Query) queue.Query { + var ( + match openapi.TextMatchingStrategy + statuses []openapi.Status + ) + switch q.Match { + case "exact": + match = openapi.EXACT + case "iexact": + match = openapi.IEXACT + case "partial": + match = openapi.PARTIAL + case "ipartial": + match = openapi.IPARTIAL + } + if len(q.Status) != 0 { + for _, p := range strings.Split(q.Status, ",") { + var s openapi.Status + switch p { + case "queued": + s = openapi.QUEUED + case "pinning": + s = openapi.PINNING + case "pinned": + s = openapi.PINNED + case "failed": + s = openapi.FAILED + default: + continue + } + statuses = append(statuses, s) + } + } + return queue.Query{ + Cids: q.Cid, + Name: q.Name, + Match: match, + Statuses: statuses, + Before: q.Before, + After: q.After, + Limit: int(q.Limit), + } +} + +func getQueryMap(s string) map[string]string { + m := make(map[string]string) + match := queryMapRx.FindStringSubmatch(s) + if len(match) != 2 { + return m + } + for _, p := range strings.Split(match[1], " ") { + parts := strings.Split(p, ":") + if len(parts) == 2 { + m[parts[0]] = parts[1] + } + } + return m +} + +func newFailure(c *gin.Context, code int, err error) { + c.JSON(code, openapi.Failure{ + Error: openapi.FailureError{ + Reason: http.StatusText(code), + Details: err.Error(), + }, + }) +} + +func handleServiceErr(c *gin.Context, err error) { + if errors.Is(err, queue.ErrNotFound) { + newFailure(c, http.StatusNotFound, err) + return + } else if strings.Contains(err.Error(), "parsing token") { + newFailure(c, http.StatusUnauthorized, err) + return + } else if errors.Is(err, pinning.ErrPermissionDenied) { + newFailure(c, http.StatusForbidden, err) + return + } else { + newFailure(c, http.StatusInternalServerError, err) + return + } +} diff --git a/gateway/pinning_test.go b/gateway/pinning_test.go new file mode 100644 index 0000000..279984e --- /dev/null +++ b/gateway/pinning_test.go @@ -0,0 +1,552 @@ +package gateway_test + +import ( + "context" + "crypto/rand" + "fmt" + "io/ioutil" + "os" + "sync/atomic" + "testing" + "time" + + ipfsfiles "github.com/ipfs/go-ipfs-files" + httpapi "github.com/ipfs/go-ipfs-http-client" + logging "github.com/ipfs/go-log/v2" + psc "github.com/ipfs/go-pinning-service-http-client" + "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/libp2p/go-libp2p-core/crypto" + maddr "github.com/multiformats/go-multiaddr" + "github.com/oklog/ulid/v2" + "github.com/phayes/freeport" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/textileio/go-buckets" + "github.com/textileio/go-buckets/api/apitest" + "github.com/textileio/go-buckets/api/common" + "github.com/textileio/go-buckets/cmd" + . "github.com/textileio/go-buckets/gateway" + "github.com/textileio/go-buckets/ipns" + "github.com/textileio/go-buckets/pinning" + "github.com/textileio/go-buckets/pinning/queue" + dbc "github.com/textileio/go-threads/api/client" + "github.com/textileio/go-threads/core/did" + "github.com/textileio/go-threads/core/thread" + tdb "github.com/textileio/go-threads/db" + nc "github.com/textileio/go-threads/net/api/client" + "github.com/textileio/go-threads/util" + "golang.org/x/sync/errgroup" +) + +var ( + origins []maddr.Multiaddr + statusAll = []psc.Status{psc.StatusQueued, psc.StatusPinning, psc.StatusPinned, psc.StatusFailed} +) + +func init() { + if err := util.SetLogLevels(map[string]logging.LogLevel{ + "buckets": logging.LevelDebug, + "buckets/ps": logging.LevelDebug, + "buckets/ps-queue": logging.LevelDebug, + "buckets/gateway": logging.LevelDebug, + }); err != nil { + panic(err) + } +} + +func TestMain(m *testing.M) { + cleanup := func() {} + if os.Getenv("SKIP_SERVICES") != "true" { + cleanup = apitest.StartServices() + } + + var err error + origins, err = getOrigins() + if err != nil { + panic(fmt.Errorf("failed to get ipfs node origins: %v", err)) + } + + exitVal := m.Run() + cleanup() + os.Exit(exitVal) +} + +func Test_ListPins(t *testing.T) { + queue.MaxConcurrency = 10 // Reduce concurrency to test overloading workers + pinning.PinTimeout = time.Second * 10 + gw := newGateway(t) + + t.Run("pagination", func(t *testing.T) { + numBatches := 3 + batchSize := 10 // Must be an even number + total := numBatches * batchSize + + files := make([]path.Resolved, total) + for i := 0; i < total; i++ { + files[i] = createIpfsFile(t, i%(batchSize/2) == 0) // Two per batch should fail (blocks unavailable) + t.Logf("created file %d", i) + } + + // Blast a bunch of requests. Each batch hits a different bucket. + var done int32 + clients := make([]*psc.Client, numBatches) + for b := 0; b < numBatches; b++ { + c := newClient(t, gw) // New client and bucket + clients[b] = c + go func(c *psc.Client, b int) { + eg, gctx := errgroup.WithContext(context.Background()) + for i := 0; i < batchSize; i++ { + i := i + j := i + (b * batchSize) + f := files[j] + time.Sleep(time.Second) + eg.Go(func() error { + if gctx.Err() != nil { + return nil + } + _, err := c.Add(gctx, f.Cid(), psc.PinOpts.WithOrigins(origins...)) + atomic.AddInt32(&done, 1) + return err + }) + } + err := eg.Wait() + require.NoError(t, err) + }(c, b) + } + + time.Sleep(time.Second * 30) // Allow time for requests to be added + + // Test pagination + for _, c := range clients { + // Get new page + res1, err := c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(statusAll...), + psc.PinOpts.Limit(batchSize/2), + ) + require.NoError(t, err) + assert.Len(t, res1, batchSize/2) + + // Get next newest + res2, err := c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(statusAll...), + psc.PinOpts.FilterBefore(res1[len(res1)-1].GetCreated()), + psc.PinOpts.Limit(batchSize/2), + ) + require.NoError(t, err) + assert.Len(t, res2, batchSize/2) + + // Ensure order is decending + all := append(res1, res2...) + for i := 0; i < len(res1)-1; i++ { + assert.Greater(t, ulid.Timestamp(all[i].GetCreated()), ulid.Timestamp(all[i+1].GetCreated())) + } + + // Get oldest page in reverse + res3, err := c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(statusAll...), + psc.PinOpts.FilterAfter(time.Now().Add(-time.Hour)), // Far back in the past + psc.PinOpts.Limit(batchSize/2), + ) + require.NoError(t, err) + assert.Len(t, res3, batchSize/2) + + // Get next oldest (first page in reverse) + res4, err := c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(statusAll...), + psc.PinOpts.FilterAfter(res3[len(res3)-1].GetCreated()), + psc.PinOpts.Limit(batchSize/2), + ) + require.NoError(t, err) + assert.Len(t, res4, batchSize/2) + + // Ensure order is ascending + all = append(res3, res4...) + for i := 0; i < len(all)-1; i++ { + assert.Less(t, ulid.Timestamp(all[i].GetCreated()), ulid.Timestamp(all[i+1].GetCreated())) + } + } + + // Wait for all to complete + assert.Eventually(t, func() bool { + // Check if all requests have been sent + if atomic.LoadInt32(&done) != int32(total) { + return false + } + // Check if all request have completed + for _, c := range clients { + res, err := c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusQueued, psc.StatusPinning), + psc.PinOpts.Limit(batchSize), + ) + require.NoError(t, err) + if len(res) != 0 { + return false + } + } + return true + }, time.Minute*10, time.Second*5) + + // Test expected status counts + for _, c := range clients { + res, err := c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusPinned), + psc.PinOpts.Limit(batchSize), + ) + require.NoError(t, err) + assert.Len(t, res, batchSize-2) + + res, err = c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusFailed), + psc.PinOpts.Limit(batchSize), + ) + require.NoError(t, err) + assert.Len(t, res, 2) + } + }) + + // A comprehensive filter test is in pinning/queue/queue_test.go. + // Here we just make sure the basics are functional. + t.Run("filters", func(t *testing.T) { + c := newClient(t, gw) + + file1 := createIpfsFile(t, false) + file2 := createIpfsFile(t, false) + file3 := createIpfsFile(t, true) + + _, err := c.Add( + context.Background(), + file1.Cid(), + psc.PinOpts.WithOrigins(origins...), + psc.PinOpts.WithName("one"), + psc.PinOpts.AddMeta(map[string]string{ + "color": "blue", + }), + ) + require.NoError(t, err) + + _, err = c.Add( + context.Background(), + file2.Cid(), + psc.PinOpts.WithOrigins(origins...), + psc.PinOpts.WithName("two"), + psc.PinOpts.AddMeta(map[string]string{ + "name": "mike", + }), + ) + require.NoError(t, err) + + time.Sleep(time.Second * 10) // Allow to succeed + + // No cid match + res, err := c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusPinned), + psc.PinOpts.FilterCIDs(file3.Cid()), + ) + require.NoError(t, err) + assert.Len(t, res, 0) + + // Cid match + res, err = c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusPinned), + psc.PinOpts.FilterCIDs(file1.Cid()), + ) + require.NoError(t, err) + assert.Len(t, res, 1) + + // No name match + res, err = c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusPinned), + psc.PinOpts.FilterName("three"), + ) + require.NoError(t, err) + assert.Len(t, res, 0) + + // Name match + res, err = c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusPinned), + psc.PinOpts.FilterName("one"), + ) + require.NoError(t, err) + assert.Len(t, res, 1) + + // No meta match + res, err = c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusPinned), + psc.PinOpts.LsMeta(map[string]string{ + "color": "red", + "name": "joe", + }), + ) + require.NoError(t, err) + assert.Len(t, res, 0) + + // Meta match + res, err = c.LsSync(context.Background(), + psc.PinOpts.FilterStatus(psc.StatusPinned), + psc.PinOpts.LsMeta(map[string]string{ + "name": "mike", + }), + ) + require.NoError(t, err) + assert.Len(t, res, 1) + }) +} + +func Test_AddPin(t *testing.T) { + pinning.PinTimeout = time.Second * 5 + gw := newGateway(t) + c := newClient(t, gw) + + t.Run("add unavailable pin should fail", func(t *testing.T) { + t.Parallel() + folder := createIpfsFolder(t, true) + res, err := c.Add(context.Background(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + assert.NotEmpty(t, res.GetRequestId()) + assert.NotEmpty(t, res.GetCreated()) + assert.Equal(t, psc.StatusQueued, res.GetStatus()) + + time.Sleep(time.Second * 10) // Allow to fail + + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + assert.Equal(t, psc.StatusFailed, res.GetStatus()) + }) + + t.Run("add available pin should succeed", func(t *testing.T) { + t.Parallel() + folder := createIpfsFolder(t, false) + res, err := c.Add(context.Background(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + assert.NotEmpty(t, res.GetRequestId()) + assert.NotEmpty(t, res.GetCreated()) + assert.Equal(t, psc.StatusQueued, res.GetStatus()) + + time.Sleep(time.Second * 10) // Allow to succeed + + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + assert.Equal(t, psc.StatusPinned, res.GetStatus()) + assert.True(t, res.GetPin().GetCid().Equals(folder.Cid())) + }) +} + +func Test_GetPin(t *testing.T) { + pinning.PinTimeout = time.Second * 5 + gw := newGateway(t) + c := newClient(t, gw) + + t.Run("get nonexistent pin should fail", func(t *testing.T) { + t.Parallel() + _, err := c.GetStatusByID(context.Background(), ulid.MustNew(123, rand.Reader).String()) + require.Error(t, err) + }) + + t.Run("get bad pin should report correct status at each stage", func(t *testing.T) { + t.Parallel() + folder := createIpfsFolder(t, true) + res, err := c.Add(context.Background(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + assert.Equal(t, psc.StatusQueued, res.GetStatus()) + + assert.Eventually(t, func() bool { + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + return res.GetStatus() == psc.StatusPinning + }, time.Second*10, time.Millisecond*200) + + assert.Eventually(t, func() bool { + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + return res.GetStatus() == psc.StatusFailed + }, time.Second*10, time.Millisecond*200) + }) + + t.Run("get good pin should report correct status at each stage", func(t *testing.T) { + t.Parallel() + folder := createIpfsFolder(t, false) + res, err := c.Add(context.Background(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + assert.Equal(t, psc.StatusQueued, res.GetStatus()) + + assert.Eventually(t, func() bool { + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + return res.GetStatus() == psc.StatusPinning + }, time.Second*10, time.Millisecond*200) + + assert.Eventually(t, func() bool { + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + return res.GetStatus() == psc.StatusPinned + }, time.Second*10, time.Millisecond*200) + }) +} + +func Test_ReplacePin(t *testing.T) { + pinning.PinTimeout = time.Second * 5 + gw := newGateway(t) + c := newClient(t, gw) + + folder := createIpfsFolder(t, false) + res, err := c.Add(context.Background(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + + time.Sleep(time.Second * 10) // Allow to succeed + + t.Run("replace with unavailable pin should fail", func(t *testing.T) { + folder := createIpfsFolder(t, true) + res2, err := c.Replace(context.Background(), res.GetRequestId(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + assert.Equal(t, res.GetRequestId(), res2.GetRequestId()) + assert.Equal(t, psc.StatusQueued, res2.GetStatus()) + + time.Sleep(time.Second * 10) // Allow to fail + + res3, err := c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + assert.Equal(t, psc.StatusFailed, res3.GetStatus()) + }) + + t.Run("replace with available pin should succeed", func(t *testing.T) { + folder := createIpfsFolder(t, false) + res2, err := c.Replace(context.Background(), res.GetRequestId(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + assert.Equal(t, res.GetRequestId(), res2.GetRequestId()) + assert.Equal(t, psc.StatusQueued, res2.GetStatus()) + + time.Sleep(time.Second * 10) // Allow to succeed + + res3, err := c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + assert.Equal(t, psc.StatusPinned, res3.GetStatus()) + assert.True(t, res3.GetPin().GetCid().Equals(folder.Cid())) + }) +} + +func Test_RemovePin(t *testing.T) { + pinning.PinTimeout = time.Second * 5 + gw := newGateway(t) + c := newClient(t, gw) + + folder := createIpfsFolder(t, false) + res, err := c.Add(context.Background(), folder.Cid(), psc.PinOpts.WithOrigins(origins...)) + require.NoError(t, err) + + time.Sleep(time.Second * 10) // Allow to succeed + + err = c.DeleteByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.NoError(t, err) + assert.Equal(t, psc.StatusPinning, res.GetStatus()) // Removal should start right away + + time.Sleep(time.Second * 5) // Allow to succeed + + res, err = c.GetStatusByID(context.Background(), res.GetRequestId()) + require.Error(t, err) +} + +func newGateway(t *testing.T) *Gateway { + threadsAddr := apitest.GetThreadsApiAddr() + net, err := nc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...) + require.NoError(t, err) + + db, err := dbc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...) + require.NoError(t, err) + ipfs, err := httpapi.NewApi(apitest.GetIPFSApiMultiAddr()) + require.NoError(t, err) + ipnsms := tdb.NewTxMapDatastore() + ipnsm, err := ipns.NewManager(ipnsms, ipfs) + require.NoError(t, err) + lib, err := buckets.NewBuckets(net, db, ipfs, ipnsm, nil) + require.NoError(t, err) + + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + pss, err := util.NewBadgerDatastore(dir, "pinq") + require.NoError(t, err) + ps, err := pinning.NewService(lib, pss) + require.NoError(t, err) + + listenPort, err := freeport.GetFreePort() + require.NoError(t, err) + addr := fmt.Sprintf("127.0.0.1:%d", listenPort) + baseUrl := fmt.Sprintf("http://127.0.0.1:%d", listenPort) + gw, err := NewGateway(lib, ipfs, ipnsm, ps, Config{ + Addr: addr, + URL: baseUrl, + }) + cmd.ErrCheck(err) + gw.Start() + + t.Cleanup(func() { + require.NoError(t, gw.Close()) + require.NoError(t, ps.Close()) + require.NoError(t, pss.Close()) + require.NoError(t, lib.Close()) + require.NoError(t, ipnsm.Close()) + require.NoError(t, ipnsms.Close()) + require.NoError(t, db.Close()) + require.NoError(t, net.Close()) + }) + return gw +} + +func newClient(t *testing.T, gw *Gateway) *psc.Client { + token := newIdentityToken(t) + buck, _, _, err := gw.Buckets().Create(context.Background(), token) + require.NoError(t, err) + url := fmt.Sprintf("%s/bps/%s", gw.Url(), buck.Key) + return psc.NewClient(url, string(token)) +} + +func newIdentityToken(t *testing.T) did.Token { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + id := thread.NewLibp2pIdentity(sk) + token, err := id.Token("did:key:foo", time.Hour) + require.NoError(t, err) + return token +} + +func createIpfsFile(t *testing.T, hashOnly bool) (pth path.Resolved) { + ipfs, err := httpapi.NewApi(apitest.GetIPFSApiMultiAddr()) + require.NoError(t, err) + pth, err = ipfs.Unixfs().Add( + context.Background(), + ipfsfiles.NewMapDirectory(map[string]ipfsfiles.Node{ + "file.txt": ipfsfiles.NewBytesFile(util.GenerateRandomBytes(512)), + }), + options.Unixfs.HashOnly(hashOnly), + ) + require.NoError(t, err) + return pth +} + +func createIpfsFolder(t *testing.T, hashOnly bool) (pth path.Resolved) { + ipfs, err := httpapi.NewApi(apitest.GetIPFSApiMultiAddr()) + require.NoError(t, err) + pth, err = ipfs.Unixfs().Add( + context.Background(), + ipfsfiles.NewMapDirectory(map[string]ipfsfiles.Node{ + "file1.txt": ipfsfiles.NewBytesFile(util.GenerateRandomBytes(1024)), + "folder1": ipfsfiles.NewMapDirectory(map[string]ipfsfiles.Node{ + "file2.txt": ipfsfiles.NewBytesFile(util.GenerateRandomBytes(512)), + }), + }), + options.Unixfs.HashOnly(hashOnly), + ) + require.NoError(t, err) + return pth +} + +func getOrigins() ([]maddr.Multiaddr, error) { + ipfs, err := httpapi.NewApi(apitest.GetIPFSApiMultiAddr()) + if err != nil { + return nil, err + } + return pinning.GetLocalAddrs(ipfs) +} diff --git a/gateway/push.go b/gateway/push.go new file mode 100644 index 0000000..a6ccb67 --- /dev/null +++ b/gateway/push.go @@ -0,0 +1,183 @@ +package gateway + +import ( + "context" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/textileio/go-buckets" + "github.com/textileio/go-buckets/dag" + core "github.com/textileio/go-threads/core/thread" +) + +// UploadTimeout is the max time taken to push files to a bucket. +var UploadTimeout = time.Hour + +const chunkSize = 1024 * 32 + +// PostError wraps errors as JSON. +type PostError struct { + Error string `json:"error"` +} + +type chanErr struct { + code int + err error +} + +// PushPathsResult wraps a single path result. +type PushPathsResult struct { + Path string `json:"path"` + Cid string `json:"cid"` + Size int64 `json:"size"` +} + +// PushPathsResults wraps all path results. +type PushPathsResults struct { + Results []PushPathsResult `json:"results"` + Pinned int64 `json:"pinned"` + Bucket *buckets.Bucket `json:"bucket"` +} + +// bucketPushPathsHandler handles bucket pushes over HTTP. +// +// For example: +// > curl -H "Authorization: Bearer " \ +// -F push=@ \ +// -F push=@ \ +// http://127.0.0.1:8000/b/ +// +// To ensure updates are fast-forward-only, use the root query param on the request URL: +// ?root=/ipfs/ +func (g *Gateway) bucketPushPathsHandler(c *gin.Context) { + thread, err := g.getThread(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ + Error: err.Error(), + }) + return + } + g.pushBucketPaths(c, thread, c.Param("key")) +} + +func (g *Gateway) pushBucketPaths(c *gin.Context, thread core.ID, key string) { + token, ok := getAuth(c) + if !ok { + c.AbortWithStatusJSON(http.StatusUnauthorized, PostError{ + Error: fmt.Sprintf("authorization required"), + }) + return + } + var root path.Resolved + if v, ok := c.GetQuery("root"); ok { + var err error + root, err = dag.NewResolvedPath(v) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ + Error: fmt.Sprintf("parsing root param: %v", err), + }) + return + } + } + + _, params, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ + Error: fmt.Sprintf("parsing content-type: %v", err), + }) + return + } + boundary, ok := params["boundary"] + if !ok { + c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ + Error: "invalid multipart boundary", + }) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), UploadTimeout) + defer cancel() + in, out, errs := g.lib.PushPaths(ctx, thread, key, token, root) + if len(errs) != 0 { + err := <-errs + c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ + Error: fmt.Sprintf("starting push: %v", err), + }) + return + } + + errCh := make(chan chanErr) + go func() { + defer close(in) + mr := multipart.NewReader(c.Request.Body, boundary) + buf := make([]byte, chunkSize) + for { + part, err := mr.NextPart() + if err == io.EOF { + return + } else if err != nil { + errCh <- chanErr{ + code: http.StatusInternalServerError, + err: fmt.Errorf("reading part: %v", err), + } + return + } + for { + n, err := part.Read(buf) + input := buckets.PushPathsInput{ + Path: part.FileName(), + } + if n > 0 { + input.Chunk = make([]byte, n) + copy(input.Chunk, buf[:n]) + in <- input + } else if err == io.EOF { + in <- input + part.Close() + break + } else if err != nil { + errCh <- chanErr{ + code: http.StatusInternalServerError, + err: fmt.Errorf("reading part: %v", err), + } + part.Close() + return + } + } + } + }() + + results := PushPathsResults{} + for { + select { + case res := <-out: + results.Results = append(results.Results, PushPathsResult{ + Path: res.Path, + Cid: res.Cid.String(), + Size: res.Size, + }) + results.Bucket = res.Bucket + results.Pinned = res.Pinned + case err := <-errs: + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, PostError{ + Error: err.Error(), + }) + } else { + c.JSON(http.StatusCreated, results) + } + return + case err := <-errCh: + c.AbortWithStatusJSON(err.code, PostError{ + Error: err.err.Error(), + }) + return + } + } +} diff --git a/gateway/push_test.go b/gateway/push_test.go new file mode 100644 index 0000000..a4ec6a2 --- /dev/null +++ b/gateway/push_test.go @@ -0,0 +1,98 @@ +package gateway_test + +import ( + "bytes" + "context" + "crypto/rand" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/textileio/go-threads/core/did" +) + +func Test_PushBucketPaths(t *testing.T) { + gw := newGateway(t) + + token := newIdentityToken(t) + buck, _, _, err := gw.Buckets().Create(context.Background(), token) + require.NoError(t, err) + + files := make(map[string]*os.File) + files["file"] = getRandomFile(t, 1024) + files["dir/file"] = getRandomFile(t, 1024) + files["dir/dir/file"] = getRandomFile(t, 1024) + + url := fmt.Sprintf("%s/thread/%s/buckets/%s?root=%s", gw.Url(), buck.Thread, buck.Key, buck.Path) + pushBucketPaths(t, url, token, files) + + item, _, err := gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "") + require.NoError(t, err) + assert.True(t, item.IsDir) + assert.Len(t, item.Items, 3) // .textileseed, file, dir + + item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir") + require.NoError(t, err) + assert.True(t, item.IsDir) + assert.Len(t, item.Items, 2) // file, dir + + item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir/dir") + require.NoError(t, err) + assert.True(t, item.IsDir) + assert.Len(t, item.Items, 1) // file + + item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "file") + require.NoError(t, err) + assert.False(t, item.IsDir) + assert.Equal(t, 1024, int(item.Size)) + + item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir/file") + require.NoError(t, err) + assert.False(t, item.IsDir) + assert.Equal(t, 1024, int(item.Size)) + + item, _, err = gw.Buckets().ListPath(context.Background(), buck.Thread, buck.Key, token, "dir/dir/file") + require.NoError(t, err) + assert.False(t, item.IsDir) + assert.Equal(t, 1024, int(item.Size)) +} + +func getRandomFile(t *testing.T, size int64) *os.File { + tmp, err := ioutil.TempFile("", "") + require.NoError(t, err) + _, err = io.CopyN(tmp, rand.Reader, size) + require.NoError(t, err) + _, err = tmp.Seek(0, 0) + require.NoError(t, err) + return tmp +} + +func pushBucketPaths(t *testing.T, url string, token did.Token, files map[string]*os.File) { + var b bytes.Buffer + w := multipart.NewWriter(&b) + for pth, f := range files { + fw, err := w.CreateFormFile("push", pth) + require.NoError(t, err) + _, err = io.Copy(fw, f) + require.NoError(t, err) + } + err := w.Close() + require.NoError(t, err) + + req, err := http.NewRequest("POST", url, &b) + require.NoError(t, err) + + req.Header.Set("Content-Type", w.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+string(token)) + + c := &http.Client{} + res, err := c.Do(req) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, res.StatusCode) +} diff --git a/gateway/threads.go b/gateway/threads.go index a64bdfd..9180388 100644 --- a/gateway/threads.go +++ b/gateway/threads.go @@ -2,81 +2,60 @@ package gateway import ( "context" - "fmt" + "errors" "net/http" - "strings" + gopath "path" "github.com/gin-gonic/gin" - col "github.com/textileio/go-buckets/collection" + "github.com/textileio/go-buckets/collection" "github.com/textileio/go-threads/core/did" - "github.com/textileio/go-threads/core/thread" - "github.com/textileio/go-threads/db" + core "github.com/textileio/go-threads/core/thread" ) -// collectionHandler handles collection requests. -func (g *Gateway) collectionHandler(c *gin.Context) { - threadID, err := thread.Decode(c.Param("thread")) +func (g *Gateway) threadHandler(c *gin.Context) { + thread, err := core.Decode(c.Param("thread")) if err != nil { - renderError(c, http.StatusBadRequest, fmt.Errorf("invalid thread ID")) + renderError(c, http.StatusBadRequest, errors.New("invalid thread ID")) return } - g.renderCollection(c, threadID, c.Param("collection")) + g.renderThread(c, thread) } -// renderCollection renders all instances in a collection. -func (g *Gateway) renderCollection(c *gin.Context, threadID thread.ID, collection string) { - ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) - defer cancel() +func (g *Gateway) renderThread(c *gin.Context, thread core.ID) { token := did.Token(c.Query("token")) - jsn := c.Query("json") == "true" - if collection == col.Name && !jsn { - g.renderBucket(c, ctx, threadID, token) - return - } else { - var dummy interface{} - res, err := g.lib.DB().Find(ctx, threadID, collection, &db.Query{}, &dummy, db.WithTxnToken(token)) - if err != nil { - render404(c) - return - } - c.JSON(http.StatusOK, res) - } -} - -// instanceHandler handles collection instance requests. -func (g *Gateway) instanceHandler(c *gin.Context) { - threadID, err := thread.Decode(c.Param("thread")) + ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) + defer cancel() + rep, err := g.lib.List(ctx, thread, token) if err != nil { - renderError(c, http.StatusBadRequest, fmt.Errorf("invalid thread ID")) - return - } - g.renderInstance(c, threadID, c.Param("collection"), c.Param("id"), c.Param("path")) -} - -// renderInstance renders an instance in a collection. -// If the collection is buckets, the built-in buckets UI in rendered instead. -// This can be overridden with the query param json=true. -func (g *Gateway) renderInstance(c *gin.Context, threadID thread.ID, collection, id, pth string) { - pth = strings.TrimPrefix(pth, "/") - jsn := c.Query("json") == "true" - if (collection != col.Name || jsn) && pth != "" { render404(c) return } - token := did.Token(c.Query("token")) - - ctx, cancel := context.WithTimeout(context.Background(), handlerTimeout) - defer cancel() - if collection == col.Name && !jsn { - g.renderBucketPath(c, ctx, threadID, id, pth, token) - return - } else { - var res interface{} - if err := g.lib.DB().FindByID(ctx, threadID, collection, id, &res, db.WithTxnToken(token)); err != nil { - render404(c) - return + links := make([]link, len(rep)) + for i, r := range rep { + var name string + if r.Name != "" { + name = r.Name + } else { + name = r.Key + } + p := gopath.Join("thread", thread.String(), collection.Name, r.Key) + if token.Defined() { + p += "?token=" + string(token) + } + links[i] = link{ + Name: name, + Path: p, + Size: "", + Links: "", } - c.JSON(http.StatusOK, res) } + c.HTML(http.StatusOK, "/public/html/unixfs.gohtml", gin.H{ + "Title": "Index of " + gopath.Join("/thread", thread.String(), collection.Name), + "Root": "/", + "Path": "", + "Updated": "", + "Back": "", + "Links": links, + }) } diff --git a/go.mod b/go.mod index d6c607f..089accf 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/textileio/go-buckets go 1.16 -// replace github.com/textileio/go-threads => ../go-threads - require ( github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 github.com/aws/aws-sdk-go v1.32.11 // indirect @@ -11,7 +9,6 @@ require ( github.com/cheggaaa/pb/v3 v3.0.5 github.com/cloudflare/cloudflare-go v0.11.6 github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect github.com/fatih/color v1.9.0 // indirect github.com/gin-contrib/location v0.0.2 github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2 @@ -26,7 +23,6 @@ require ( github.com/ipfs/go-blockservice v0.1.4 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-datastore v0.4.5 - github.com/ipfs/go-ds-badger v0.2.6 github.com/ipfs/go-ds-flatfs v0.4.4 github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipfs/go-ipfs-chunker v0.0.5 @@ -38,6 +34,7 @@ require ( github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4 github.com/ipfs/go-merkledag v0.3.2 + github.com/ipfs/go-pinning-service-http-client v0.1.0 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/interface-go-ipfs-core v0.4.0 github.com/jbenet/go-is-domain v1.0.3 @@ -55,6 +52,7 @@ require ( github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multihash v0.0.14 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/oklog/ulid/v2 v2.0.2 github.com/olekukonko/tablewriter v0.0.4 github.com/onsi/ginkgo v1.14.0 // indirect github.com/pelletier/go-toml v1.7.0 // indirect @@ -65,26 +63,30 @@ require ( github.com/smartystreets/assertions v1.0.1 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cast v1.3.1 // indirect - github.com/spf13/cobra v1.1.1 + github.com/spf13/cobra v1.1.3 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 github.com/textileio/dcrypto v0.0.1 github.com/textileio/go-assets v0.0.0-20200430191519-b341e634e2b7 + github.com/textileio/go-datastore-extensions v1.0.1 + github.com/textileio/go-ds-badger3 v0.0.0-20210324034212-7b7fb3be3d1c github.com/textileio/go-ds-mongo v0.1.5-0.20201230201018-2b7fdca787a5 - github.com/textileio/go-threads v1.0.3-0.20210310071259-b3c756e6c9c2 + github.com/textileio/go-threads v1.0.3-0.20210331042803-3a1b7e46e91f github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2 // indirect github.com/xdg/stringprep v1.0.0 // indirect go.mongodb.org/mongo-driver v1.4.1 // indirect go.opencensus.io v0.22.6 // indirect go.uber.org/multierr v1.6.0 // indirect + golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect + golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect golang.org/x/sync v0.0.0-20201207232520-09787c993a3a - golang.org/x/sys v0.0.0-20201218084310-7d0127a74742 // indirect - golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect - golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3 // indirect + golang.org/x/sys v0.0.0-20210324051608-47abb6519492 // indirect + golang.org/x/text v0.3.5 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad // indirect google.golang.org/grpc v1.35.0 google.golang.org/protobuf v1.25.0 @@ -92,3 +94,5 @@ require ( gopkg.in/ini.v1 v1.55.0 // indirect honnef.co/go/tools v0.0.1-2020.1.4 // indirect ) + +replace github.com/ipfs/go-pinning-service-http-client => github.com/textileio/go-pinning-service-http-client v0.1.1-0.20210328174252-bc12e73b9a56 diff --git a/go.sum b/go.sum index 0f48d30..198c809 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOv github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -94,11 +96,13 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -116,9 +120,11 @@ github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6ps github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/badger/v3 v3.2011.1 h1:Hmyof0WMEF/QtutX5SQHzIMnJQxb/IrSzhjckV2SD6g= +github.com/dgraph-io/badger/v3 v3.2011.1/go.mod h1:0rLLrQpKVQAL0or/lBLMQznhr6dWWX7h5AKnmnqx268= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d h1:eQYOG6A4td1tht0NdJB9Ls6DsXRGb2Ft6X9REU/MbbE= +github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d/go.mod h1:tv2ec8nA7vRpSYX7/MbP52ihrUMXIHit54CQMq8npXQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -160,6 +166,7 @@ github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmC github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-delve/delve v1.5.0/go.mod h1:c6b3a1Gry6x8a4LGCe/CWzrocrfaHvkUxCj3k4bvSUQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -213,8 +220,9 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -245,6 +253,8 @@ github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZV github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.0 h1:/PtAHvnBY4Kqnx/xCQ3OIV9uYcSFGScBsWI3Oogeh6w= +github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -253,6 +263,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= @@ -509,12 +520,14 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -831,6 +844,7 @@ github.com/manifoldco/promptui v0.7.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEX github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= +github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -873,6 +887,7 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.0 h1:iDwIio/3gk2QtLLEsqU5lInaMzos0hDTz8a6lazSFVw= github.com/mitchellh/mapstructure v1.3.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcloughlin/avo v0.0.0-20201105074841-5d2f697d268f/go.mod h1:6aKT4zZIrpGqB3RpFU14ByCSSyKY6LfJz4J/JJChHfI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -985,6 +1000,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1033,6 +1049,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -1056,13 +1073,15 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -1090,13 +1109,15 @@ github.com/textileio/go-assets v0.0.0-20200430191519-b341e634e2b7 h1:J7+UXJT/Ku8 github.com/textileio/go-assets v0.0.0-20200430191519-b341e634e2b7/go.mod h1:j7aKMh8sbbtvttp7V7yCOkHW/pfRtIM/6h+8qEDsLyI= github.com/textileio/go-datastore-extensions v1.0.1 h1:qIJGqJaigQ1wD4TdwS/hf73u0HChhXvvUSJuxBEKS+c= github.com/textileio/go-datastore-extensions v1.0.1/go.mod h1:Pzj9FDRkb55910dr/FX8M7WywvnS26gBgEDez1ZBuLE= -github.com/textileio/go-ds-badger v0.2.7-0.20201204225019-4ee78c4a40e2 h1:VIWkccJp5k8x1hbwQsSn6Xnns4wUDEW6XdiMLiOpr7k= -github.com/textileio/go-ds-badger v0.2.7-0.20201204225019-4ee78c4a40e2/go.mod h1:qEZ/z1KyoRhGS5MYEbIcWUCCPd/0HxCkFDVeJgP1RcI= +github.com/textileio/go-ds-badger3 v0.0.0-20210324034212-7b7fb3be3d1c h1:iOWa9/AXFssCi7VhbEvVkpCYXqq/n3E2zKH6u+9pIfM= +github.com/textileio/go-ds-badger3 v0.0.0-20210324034212-7b7fb3be3d1c/go.mod h1:VX3oCNk7szANGG+baANuhTx7dheWiEPH7lp1n69ef4M= github.com/textileio/go-ds-mongo v0.1.4/go.mod h1:Zf6JlMPiIQUUmGlFFn5Z65C9p9LAvPg7XvX+qdGmTsU= github.com/textileio/go-ds-mongo v0.1.5-0.20201230201018-2b7fdca787a5 h1:wy2WAFw2u0RAh7Mnlnh7/Hmc9LlSADDHLlPNZQDBvmM= github.com/textileio/go-ds-mongo v0.1.5-0.20201230201018-2b7fdca787a5/go.mod h1:Zf6JlMPiIQUUmGlFFn5Z65C9p9LAvPg7XvX+qdGmTsU= -github.com/textileio/go-threads v1.0.3-0.20210310071259-b3c756e6c9c2 h1:ZhLGaXohsDIwi8EkggerRiYKPwWoqw+22Ny1Xbkl6zA= -github.com/textileio/go-threads v1.0.3-0.20210310071259-b3c756e6c9c2/go.mod h1:feZtQ59yf9ycYNS+L0Xk9oxL30lCTIHlWsHYbaJa45w= +github.com/textileio/go-pinning-service-http-client v0.1.1-0.20210328174252-bc12e73b9a56 h1:N5fTfazIM6XCKO7BB4RFP0dfOyjATrhsGUTl+jsq9rs= +github.com/textileio/go-pinning-service-http-client v0.1.1-0.20210328174252-bc12e73b9a56/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= +github.com/textileio/go-threads v1.0.3-0.20210331042803-3a1b7e46e91f h1:Pk/F816XNY0zHMLGTXor4CFjNY4DMUzjhhOMwm4HH6w= +github.com/textileio/go-threads v1.0.3-0.20210331042803-3a1b7e46e91f/go.mod h1:vd6zZosJtOUC3w0M2x/8j1n7PAiAze+SzAciCkn5rrI= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= github.com/tidwall/gjson v1.3.5 h1:2oW9FBNu8qt9jy5URgrzsVx/T/KSn3qn/smJQ0crlDQ= github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= @@ -1107,6 +1128,7 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.0/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -1154,7 +1176,8 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.4.0/go.mod h1:llVBH2pkj9HywK0Dtdt6lDikOjFLbceHVu/Rc0iMKLs= go.mongodb.org/mongo-driver v1.4.1 h1:38NSAyDPagwnFpUA/D5SFgbugUYR3NzYRNa4Qk9UxKs= @@ -1165,8 +1188,10 @@ go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6 h1:BdkrbWrzDlV9dnbzoP7sfN+dHheJ4J9JOaYxcUDL+ok= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= @@ -1184,6 +1209,8 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1208,8 +1235,9 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c h1:9HhBz5L/UjnK9XLtiZhYAdue5BVKep3PMmS2LuPDt8k= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1235,6 +1263,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -1260,14 +1289,17 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1277,7 +1309,7 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1327,19 +1359,22 @@ golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201218084310-7d0127a74742 h1:+CBz4km/0KPU3RGTwARGh/noP3bEwtHcq+0YcBQM2JQ= -golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1368,12 +1403,15 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3 h1:r3P/5xOq/dK1991B65Oy6E1fRF/2d/fSYZJ/fXGVfJc= -golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1387,6 +1425,7 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1450,8 +1489,9 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1462,3 +1502,4 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/info.go b/info.go new file mode 100644 index 0000000..422ab03 --- /dev/null +++ b/info.go @@ -0,0 +1,119 @@ +package buckets + +import ( + "context" + "fmt" + "time" + + "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/textileio/go-buckets/collection" + "github.com/textileio/go-threads/core/did" + core "github.com/textileio/go-threads/core/thread" +) + +// PushPathInfo pushes arbitrary info/metadata to a path, +// allowing users to add application specific path metadata. +func (b *Buckets) PushPathInfo( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + root path.Resolved, + pth string, + info map[string]interface{}, +) (*Bucket, error) { + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + return nil, err + } + defer txn.Close() + return txn.PushPathInfo(ctx, root, pth, info) +} + +// PushPathInfo is Txn based PushPathInfo. +func (t *Txn) PushPathInfo( + ctx context.Context, + root path.Resolved, + pth string, + info map[string]interface{}, +) (*Bucket, error) { + pth, err := parsePath(pth) + if err != nil { + return nil, err + } + + instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) + if err != nil { + return nil, err + } + if root != nil && root.String() != instance.Path { + return nil, ErrNonFastForward + } + + md, mdPath, ok := instance.GetMetadataForPath(pth, false) + if !ok { + return nil, fmt.Errorf("could not resolve path: %s", pth) + } + var target collection.Metadata + if mdPath != pth { // If the metadata is inherited from a parent, create a new entry + target = collection.Metadata{ + Info: make(map[string]interface{}), + } + } else { + target = md + } + if target.Info == nil { + target.Info = make(map[string]interface{}) + } + + var changed bool + for k, v := range info { + if x, ok := target.Info[k]; ok && x == v { + continue + } + target.Info[k] = v + changed = true + } + if changed { + instance.UpdatedAt = time.Now().UnixNano() + target.UpdatedAt = instance.UpdatedAt + instance.Metadata[pth] = target + if err := t.b.c.Save(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { + return nil, err + } + } + + log.Debugf("pushed info for %s in %s", pth, t.key) + return instanceToBucket(t.thread, instance), nil +} + +// PullPathInfo pulls all info at a path. +func (b *Buckets) PullPathInfo( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + pth string, +) (map[string]interface{}, error) { + if err := thread.Validate(); err != nil { + return nil, fmt.Errorf("invalid thread id: %v", err) + } + pth, err := parsePath(pth) + if err != nil { + return nil, err + } + instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) + if err != nil { + return nil, err + } + md, _, ok := instance.GetMetadataForPath(pth, false) + if !ok { + return nil, fmt.Errorf("could not resolve path: %s", pth) + } + if md.Info == nil { + md.Info = make(map[string]interface{}) + } + + log.Debugf("pulled info for %s in %s", pth, key) + return md.Info, nil +} diff --git a/ipns/ipns.go b/ipns/ipns.go index 124002c..ec7ca46 100644 --- a/ipns/ipns.go +++ b/ipns/ipns.go @@ -14,15 +14,13 @@ import ( "github.com/libp2p/go-libp2p-core/peer" mbase "github.com/multiformats/go-multibase" s "github.com/textileio/go-buckets/ipns/store" - "github.com/textileio/go-buckets/util" "github.com/textileio/go-threads/core/thread" + tutil "github.com/textileio/go-threads/util" ) -var log = logging.Logger("buckets-ipns") +var log = logging.Logger("buckets/ipns") const ( - // nameLen is the length of the random IPNS key name. - nameLen = 16 // publishTimeout publishTimeout = time.Minute * 2 // maxCancelPublishTries is the number of time cancelling a publish is allowed to fail. @@ -35,10 +33,10 @@ type Manager struct { keyAPI iface.KeyAPI nameAPI iface.NameAPI - sync.Mutex keyLocks map[string]chan struct{} ctxsLock sync.Mutex ctxs map[string]context.CancelFunc + sync.Mutex } // NewManager returns a new IPNS manager. @@ -59,11 +57,11 @@ func (m *Manager) Store() *s.Store { // CreateKey generates and saves a new IPNS key. func (m *Manager) CreateKey(ctx context.Context, dbID thread.ID) (keyID string, err error) { - key, err := m.keyAPI.Generate(ctx, util.MakeToken(nameLen), options.Key.Type(options.RSAKey)) + key, err := m.keyAPI.Generate(ctx, tutil.MakeToken(), options.Key.Type(options.Ed25519Key)) if err != nil { return } - keyID, err = peer.ToCid(key.ID()).StringOfBase(mbase.Base32) + keyID, err = peer.ToCid(key.ID()).StringOfBase(mbase.Base36) if err != nil { return } diff --git a/ipns/store/store.go b/ipns/store/store.go index f3e98d9..b59bba5 100644 --- a/ipns/store/store.go +++ b/ipns/store/store.go @@ -30,12 +30,6 @@ func NewStore(store ds.TxnDatastore) *Store { } func (s *Store) Create(name, cid string, threadID thread.ID) error { - txn, err := s.store.NewTransaction(false) - if err != nil { - return err - } - defer txn.Discard() - var buf bytes.Buffer if err := gob.NewEncoder(&buf).Encode(Key{ Name: name, @@ -46,6 +40,12 @@ func (s *Store) Create(name, cid string, threadID thread.ID) error { return err } + txn, err := s.store.NewTransaction(false) + if err != nil { + return err + } + defer txn.Discard() + // Add key value if err := txn.Put(dsPrefix.ChildString(name), buf.Bytes()); err != nil { return err @@ -64,9 +64,12 @@ func (s *Store) Get(name string) (*Key, error) { if err != nil { return nil, err } + return decode(val) +} +func decode(v []byte) (*Key, error) { var buf bytes.Buffer - buf.Write(val) + buf.Write(v) dec := gob.NewDecoder(&buf) var key Key if err := dec.Decode(&key); err != nil { @@ -96,7 +99,11 @@ func (s *Store) Delete(name string) error { } defer txn.Discard() - key, err := s.Get(name) + val, err := txn.Get(dsPrefix.ChildString(name)) + if err != nil { + return err + } + key, err := decode(val) if err != nil { return err } diff --git a/list.go b/list.go index dd15d3f..448a78c 100644 --- a/list.go +++ b/list.go @@ -2,20 +2,26 @@ package buckets import ( "context" + "fmt" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/textileio/go-threads/core/did" core "github.com/textileio/go-threads/core/thread" ) +// ListPath lists all paths under a path. func (b *Buckets) ListPath( ctx context.Context, thread core.ID, - key, pth string, + key string, identity did.Token, + pth string, ) (*PathItem, *Bucket, error) { + if err := thread.Validate(); err != nil { + return nil, nil, fmt.Errorf("invalid thread id: %v", err) + } pth = trimSlash(pth) - instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity) + instance, bpth, err := b.getBucketAndPath(ctx, thread, key, identity, pth) if err != nil { return nil, nil, err } @@ -27,6 +33,7 @@ func (b *Buckets) ListPath( return item, instanceToBucket(thread, instance), nil } +// ListIPFSPath lists all paths under a path. func (b *Buckets) ListIPFSPath(ctx context.Context, pth string) (*PathItem, error) { log.Debugf("listed ipfs path %s", pth) return b.pathToItem(ctx, nil, path.New(pth), true) diff --git a/local/access.go b/local/access.go index 8dd834f..8425260 100644 --- a/local/access.go +++ b/local/access.go @@ -2,6 +2,7 @@ package local import ( "context" + "time" "github.com/textileio/go-buckets/collection" "github.com/textileio/go-threads/core/did" @@ -17,7 +18,7 @@ func (b *Bucket) PushPathAccessRoles( pth string, roles map[did.DID]collection.Role, ) (merged map[did.DID]collection.Role, err error) { - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } @@ -33,7 +34,7 @@ func (b *Bucket) PushPathAccessRoles( // PullPathAccessRoles returns access roles for a path. func (b *Bucket) PullPathAccessRoles(ctx context.Context, pth string) (roles map[did.DID]collection.Role, err error) { - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } diff --git a/local/add.go b/local/add.go index b877105..7d04971 100644 --- a/local/add.go +++ b/local/add.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core/path" @@ -18,7 +19,7 @@ import ( func (b *Bucket) AddRemoteCid(ctx context.Context, c cid.Cid, dest string, opts ...AddOption) error { b.Lock() defer b.Unlock() - ctx, err := b.authCtx(ctx) + ctx, err := b.authCtx(ctx, time.Hour) if err != nil { return err } diff --git a/local/bucket.go b/local/bucket.go index feeceb5..96e0dc4 100644 --- a/local/bucket.go +++ b/local/bucket.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core/options" @@ -17,7 +18,8 @@ import ( "github.com/textileio/go-buckets/api/client" "github.com/textileio/go-buckets/cmd" "github.com/textileio/go-buckets/collection" - "github.com/textileio/go-buckets/util" + "github.com/textileio/go-buckets/dag" + "github.com/textileio/go-threads/core/did" "github.com/textileio/go-threads/core/thread" ) @@ -136,7 +138,7 @@ func (b *Bucket) LocalSize() (int64, error) { // Get returns info about a bucket from the remote. func (b *Bucket) Get(ctx context.Context) (bucket buckets.Bucket, err error) { - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } @@ -181,7 +183,7 @@ func (b *Bucket) RemoteLinks(ctx context.Context, pth string) (links buckets.Lin if b.links != nil { return *b.links, nil } - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } @@ -198,31 +200,9 @@ func (b *Bucket) RemoteLinks(ctx context.Context, pth string) (links buckets.Lin return links, err } -//// DBInfo returns info about the bucket's ThreadDB. -//// This info can be used to add replicas or additional peers to the bucket. -//func (b *Bucket) DBInfo(ctx context.Context) (info db.Info, cc db.CollectionConfig, err error) { -// ctx, err = b.auth(ctx) -// if err != nil { -// return -// } -// id, err := b.Thread() -// if err != nil { -// return -// } -// info, err = b.c.Threads.GetDBInfo(ctx, id) -// if err != nil { -// return -// } -// cc, err = b.c.Threads.GetCollectionInfo(ctx, id, collection.Name) -// if err != nil { -// return -// } -// return info, cc, nil -//} - // CatRemotePath writes the content of the remote path to writer. func (b *Bucket) CatRemotePath(ctx context.Context, pth string, w io.Writer) error { - ctx, err := b.authCtx(ctx) + ctx, err := b.authCtx(ctx, time.Hour) if err != nil { return err } @@ -237,7 +217,7 @@ func (b *Bucket) CatRemotePath(ctx context.Context, pth string, w io.Writer) err func (b *Bucket) Destroy(ctx context.Context) (err error) { b.Lock() defer b.Unlock() - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } @@ -257,6 +237,16 @@ func (b *Bucket) Destroy(ctx context.Context) (err error) { return nil } +// GetIdentityToken returns a did.Token for the bucket identity valid for the given duration. +func (b *Bucket) GetIdentityToken(dur time.Duration) (did.Token, error) { + ctx, err := b.authCtx(context.TODO(), dur) + if err != nil { + return "", err + } + token, _ := did.TokenFromContext(ctx) + return token, nil +} + func (b *Bucket) loadLocalRepo(ctx context.Context, pth, name string, setCidVersion bool) error { r, err := NewRepo(pth, name, options.BalancedLayout) if err != nil { @@ -300,7 +290,7 @@ func (b *Bucket) containsPath(pth string) (c bool, err error) { } func (b *Bucket) getRemoteRoot(ctx context.Context) (cid.Cid, error) { - ctx, err := b.authCtx(ctx) + ctx, err := b.authCtx(ctx, time.Hour) if err != nil { return cid.Undef, err } @@ -312,7 +302,7 @@ func (b *Bucket) getRemoteRoot(ctx context.Context) (cid.Cid, error) { if err != nil { return cid.Undef, err } - rp, err := util.NewResolvedPath(rr.Bucket.Path) + rp, err := dag.NewResolvedPath(rr.Bucket.Path) if err != nil { return cid.Undef, err } @@ -320,10 +310,10 @@ func (b *Bucket) getRemoteRoot(ctx context.Context) (cid.Cid, error) { } // authCtx returns an identity token context for authentication and authorization. -func (b *Bucket) authCtx(ctx context.Context) (context.Context, error) { +func (b *Bucket) authCtx(ctx context.Context, dur time.Duration) (context.Context, error) { identity := &thread.Libp2pIdentity{} if err := identity.UnmarshalString(b.conf.Viper.GetString("identity")); err != nil { return nil, err } - return authCtx(ctx, b.c, identity) + return authCtx(ctx, b.c, identity, dur) } diff --git a/local/bucket_test.go b/local/bucket_test.go index fb76e32..ce8cbdf 100644 --- a/local/bucket_test.go +++ b/local/bucket_test.go @@ -76,18 +76,6 @@ func TestBucket(t *testing.T) { assert.NotEmpty(t, links.IPNS) }) - //t.Run("DBInfo", func(t *testing.T) { - // dbinfo, cc, err := buck.DBInfo(context.Background()) - // require.NoError(t, err) - // assert.True(t, dbinfo.Key.Defined()) - // assert.NotEmpty(t, dbinfo.Addrs) - // assert.NotEmpty(t, cc.Name) - // assert.NotEmpty(t, cc.Schema) - // assert.NotEmpty(t, cc.WriteValidator) - // assert.NotEmpty(t, cc.ReadFilter) - // assert.NotEmpty(t, cc.Indexes) - //}) - t.Run("Destroy", func(t *testing.T) { err = buck.Destroy(context.Background()) require.NoError(t, err) diff --git a/local/buckets.go b/local/buckets.go index 14eda29..e905af1 100644 --- a/local/buckets.go +++ b/local/buckets.go @@ -16,7 +16,7 @@ import ( "github.com/textileio/go-buckets/api/client" "github.com/textileio/go-buckets/cmd" "github.com/textileio/go-buckets/collection" - "github.com/textileio/go-buckets/util" + "github.com/textileio/go-buckets/dag" "github.com/textileio/go-threads/core/thread" ) @@ -56,7 +56,7 @@ func NewBuckets(c *client.Client, config cmd.ConfConfig) *Buckets { return &Buckets{c: c, config: config} } -// Clients returns the underlying client object. +// Client returns the underlying client object. func (b *Buckets) Client() *client.Client { return b.c } @@ -154,7 +154,7 @@ func (b *Buckets) NewBucket(ctx context.Context, conf Config, opts ...NewOption) pushBlock: make(chan struct{}, 1), } - ctx, err = authCtx(ctx, b.c, conf.Identity) + ctx, err = authCtx(ctx, b.c, conf.Identity, time.Hour) if err != nil { return nil, err } @@ -200,7 +200,7 @@ func (b *Buckets) NewBucket(ctx context.Context, conf Config, opts ...NewOption) if err = buck.repo.SetRemotePath(collection.SeedName, sc); err != nil { return nil, err } - rp, err := util.NewResolvedPath(rep.Bucket.Path) + rp, err := dag.NewResolvedPath(rep.Bucket.Path) if err != nil { return nil, err } @@ -325,7 +325,7 @@ func (b *Buckets) RemoteBuckets( id thread.ID, identity thread.Identity, ) (list []buckets.Bucket, err error) { - ctx, err = authCtx(ctx, b.c, identity) + ctx, err = authCtx(ctx, b.c, identity, time.Hour) if err != nil { return nil, err } @@ -344,6 +344,11 @@ func (b *Buckets) RemoteBuckets( } // authCtx returns an identity token context for authentication and authorization. -func authCtx(ctx context.Context, c *client.Client, identity thread.Identity) (context.Context, error) { - return c.NewTokenContext(ctx, identity, time.Second) +func authCtx( + ctx context.Context, + c *client.Client, + identity thread.Identity, + dur time.Duration, +) (context.Context, error) { + return c.NewTokenContext(ctx, identity, dur) } diff --git a/local/buckets_test.go b/local/buckets_test.go index 0652156..58f7764 100644 --- a/local/buckets_test.go +++ b/local/buckets_test.go @@ -10,11 +10,10 @@ import ( "testing" "time" - "github.com/libp2p/go-libp2p-core/crypto" - ipfsfiles "github.com/ipfs/go-ipfs-files" httpapi "github.com/ipfs/go-ipfs-http-client" "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,8 +21,8 @@ import ( "github.com/textileio/go-buckets/api/client" "github.com/textileio/go-buckets/api/common" . "github.com/textileio/go-buckets/local" - "github.com/textileio/go-buckets/util" "github.com/textileio/go-threads/core/thread" + tutil "github.com/textileio/go-threads/util" ) func TestMain(m *testing.M) { @@ -308,9 +307,9 @@ func createIpfsFolder(t *testing.T) (pth path.Resolved) { pth, err = ipfs.Unixfs().Add( context.Background(), ipfsfiles.NewMapDirectory(map[string]ipfsfiles.Node{ - "file1.txt": ipfsfiles.NewBytesFile(util.GenerateRandomBytes(1024)), + "file1.txt": ipfsfiles.NewBytesFile(tutil.GenerateRandomBytes(1024)), "folder1": ipfsfiles.NewMapDirectory(map[string]ipfsfiles.Node{ - "file2.txt": ipfsfiles.NewBytesFile(util.GenerateRandomBytes(512)), + "file2.txt": ipfsfiles.NewBytesFile(tutil.GenerateRandomBytes(512)), }), }), ) diff --git a/local/crypto.go b/local/crypto.go index 5dea8bc..5c4bbd0 100644 --- a/local/crypto.go +++ b/local/crypto.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "time" "github.com/textileio/dcrypto" ) @@ -105,7 +106,7 @@ func (b *Bucket) decryptRemotePath( fn decryptFunc, w io.Writer, ) error { - ctx, err := b.authCtx(ctx) + ctx, err := b.authCtx(ctx, time.Hour) if err != nil { return err } diff --git a/local/list.go b/local/list.go index 194534d..70d35d2 100644 --- a/local/list.go +++ b/local/list.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "time" "github.com/ipfs/go-cid" pb "github.com/textileio/go-buckets/api/pb/buckets" @@ -29,7 +30,7 @@ func (b *Bucket) ListRemotePath(ctx context.Context, pth string) (items []Bucket if pth == "." || pth == "/" || pth == "./" { pth = "" } - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } diff --git a/local/pull.go b/local/pull.go index 5d0b440..13a6297 100644 --- a/local/pull.go +++ b/local/pull.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "sync" + "time" cid "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -25,7 +26,7 @@ var MaxPullConcurrency = 10 func (b *Bucket) PullRemote(ctx context.Context, opts ...PathOption) (roots Roots, err error) { b.Lock() defer b.Unlock() - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } diff --git a/local/push.go b/local/push.go index 071655e..e3b5c46 100644 --- a/local/push.go +++ b/local/push.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "time" du "github.com/ipfs/go-merkledag/dagutils" "github.com/ipfs/interface-go-ipfs-core/path" @@ -13,12 +14,12 @@ import ( "github.com/textileio/go-threads/core/thread" ) -// PushRemote pushes local files. +// PushLocal pushes local files. // By default, only staged changes are pushed. See PathOption for more info. func (b *Bucket) PushLocal(ctx context.Context, opts ...PathOption) (roots Roots, err error) { b.Lock() defer b.Unlock() - ctx, err = b.authCtx(ctx) + ctx, err = b.authCtx(ctx, time.Hour) if err != nil { return } diff --git a/local/watch.go b/local/watch.go index 2095839..fa1f1e6 100644 --- a/local/watch.go +++ b/local/watch.go @@ -3,6 +3,7 @@ package local import ( "context" "errors" + "strings" "time" "github.com/textileio/go-buckets" @@ -128,7 +129,7 @@ func (b *Bucket) watchPush(ctx context.Context, events chan<- Event) error { }() if _, err := b.PushLocal(ctx, WithEvents(events)); errors.Is(err, ErrUpToDate) { return nil - } else if errors.Is(err, buckets.ErrNonFastForward) { + } else if err != nil && strings.Contains(err.Error(), buckets.ErrNonFastForward.Error()) { // Pull remote changes if _, err = b.PullRemote(ctx, WithEvents(events)); err != nil { return err diff --git a/move.go b/move.go index 7f6646f..0967057 100644 --- a/move.go +++ b/move.go @@ -14,22 +14,37 @@ import ( core "github.com/textileio/go-threads/core/thread" ) +// MovePath moves a path to a different location. +// The destination path does not need to exist. +// Currently, moving the root path is not possible. func (b *Buckets) MovePath( ctx context.Context, thread core.ID, - key, fpth, tpth string, + key string, identity did.Token, + root path.Resolved, + fpth, tpth string, ) (int64, *Bucket, error) { - lk := b.locks.Get(lock(key)) - lk.Acquire() - defer lk.Release() + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + return 0, nil, err + } + defer txn.Close() + return txn.MovePath(ctx, root, fpth, tpth) +} +// MovePath is Txn based MovePath. +func (t *Txn) MovePath( + ctx context.Context, + root path.Resolved, + fpth, tpth string, +) (int64, *Bucket, error) { fpth, err := parsePath(fpth) if err != nil { return 0, nil, err } if fpth == "" { - // @todo: enable move of root directory + // @todo: Enable move of root directory return 0, nil, fmt.Errorf("root cannot be moved") } tpth, err = parsePath(tpth) @@ -41,17 +56,20 @@ func (b *Buckets) MovePath( return 0, nil, fmt.Errorf("path is destination") } - instance, pth, err := b.getBucketAndPath(ctx, thread, key, fpth, identity) + instance, pth, err := t.b.getBucketAndPath(ctx, t.thread, t.key, t.identity, fpth) if err != nil { return 0, nil, fmt.Errorf("getting path: %v", err) } + if root != nil && root.String() != instance.Path { + return 0, nil, ErrNonFastForward + } instance.UpdatedAt = time.Now().UnixNano() instance.SetMetadataAtPath(tpth, collection.Metadata{ UpdatedAt: instance.UpdatedAt, }) instance.UnsetMetadataWithPrefix(fpth + "/") - if err := b.c.Verify(ctx, thread, instance, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { return 0, nil, fmt.Errorf("verifying bucket update: %v", err) } @@ -59,7 +77,7 @@ func (b *Buckets) MovePath( if err != nil { return 0, nil, err } - fitem, err := b.pathToItem(ctx, instance, fbpth, false) + fitem, err := t.b.pathToItem(ctx, instance, fbpth, false) if err != nil { return 0, nil, err } @@ -67,7 +85,7 @@ func (b *Buckets) MovePath( if err != nil { return 0, nil, err } - titem, err := b.pathToItem(ctx, instance, tbpth, false) + titem, err := t.b.pathToItem(ctx, instance, tbpth, false) if err == nil { if fitem.IsDir && !titem.IsDir { return 0, nil, fmt.Errorf("destination is not a directory") @@ -80,19 +98,19 @@ func (b *Buckets) MovePath( } } - pnode, err := dag.GetNodeAtPath(ctx, b.ipfs, pth, instance.GetLinkEncryptionKey()) + pnode, err := dag.GetNodeAtPath(ctx, t.b.ipfs, pth, instance.GetLinkEncryptionKey()) if err != nil { return 0, nil, fmt.Errorf("getting node: %v", err) } var dirPath path.Resolved if instance.IsPrivate() { - ctx, dirPath, err = dag.CopyDag(ctx, b.ipfs, instance, pnode, fpth, tpth) + ctx, dirPath, err = dag.CopyDag(ctx, t.b.ipfs, instance, pnode, fpth, tpth) if err != nil { return 0, nil, fmt.Errorf("copying node: %v", err) } } else { - ctx, dirPath, err = b.setPathFromExistingCid( + ctx, dirPath, err = t.b.setPathFromExistingCid( ctx, instance, path.New(instance.Path), @@ -110,25 +128,25 @@ func (b *Buckets) MovePath( // If "a/b" => "a/", cleanup only needed for priv if strings.HasPrefix(fpth, tpth) { if instance.IsPrivate() { - ctx, dirPath, err = b.removePath(ctx, instance, fpth) + ctx, dirPath, err = t.b.removePath(ctx, instance, fpth) if err != nil { return 0, nil, fmt.Errorf("removing path: %v", err) } instance.Path = dirPath.String() } - if err := b.saveAndPublish(ctx, thread, instance, identity); err != nil { + if err := t.b.saveAndPublish(ctx, t.thread, t.identity, instance); err != nil { return 0, nil, err } log.Debugf("moved %s to %s", fpth, tpth) - return dag.GetPinnedBytes(ctx), instanceToBucket(thread, instance), nil + return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil } if strings.HasPrefix(tpth, fpth) { // If "a/" => "a/b" cleanup each leaf in "a" that isn't "b" (skipping .textileseed) ppth := path.Join(path.New(instance.Path), fpth) - item, err := b.listPath(ctx, instance, ppth) + item, err := t.b.listPath(ctx, instance, ppth) if err != nil { return 0, nil, fmt.Errorf("listing path: %v", err) } @@ -137,7 +155,7 @@ func (b *Buckets) MovePath( if strings.Compare(chld.Name, collection.SeedName) == 0 || sp == tpth { continue } - ctx, dirPath, err = b.removePath(ctx, instance, trimSlash(sp)) + ctx, dirPath, err = t.b.removePath(ctx, instance, trimSlash(sp)) if err != nil { return 0, nil, fmt.Errorf("removing path: %v", err) } @@ -145,17 +163,17 @@ func (b *Buckets) MovePath( } } else { // if a/ => b/ remove a - ctx, dirPath, err = b.removePath(ctx, instance, fpth) + ctx, dirPath, err = t.b.removePath(ctx, instance, fpth) if err != nil { return 0, nil, fmt.Errorf("removing path: %v", err) } instance.Path = dirPath.String() } - if err := b.saveAndPublish(ctx, thread, instance, identity); err != nil { + if err := t.b.saveAndPublish(ctx, t.thread, t.identity, instance); err != nil { return 0, nil, err } log.Debugf("moved %s to %s", fpth, tpth) - return dag.GetPinnedBytes(ctx), instanceToBucket(thread, instance), nil + return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil } diff --git a/options.go b/options.go index 9734f8f..1df71e6 100644 --- a/options.go +++ b/options.go @@ -6,6 +6,7 @@ import ( core "github.com/textileio/go-threads/core/thread" ) +// CreateOptions are used when creating a new bucket. type CreateOptions struct { Thread core.ID Name string @@ -44,6 +45,7 @@ func WithCid(cid c.Cid) CreateOption { } } +// Options are used to perform bucket operations. type Options struct { Root path.Resolved Progress chan<- int64 diff --git a/path.go b/path.go index 7c313f4..f1c51e0 100644 --- a/path.go +++ b/path.go @@ -18,6 +18,7 @@ import ( core "github.com/textileio/go-threads/core/thread" ) +// parsePath validates and cleans a bucket path. func parsePath(pth string) (fpth string, err error) { if strings.Contains(pth, collection.SeedName) { err = fmt.Errorf("paths containing %s are not allowed", collection.SeedName) @@ -164,8 +165,9 @@ func (b *Buckets) removePath( func (b *Buckets) getBucketAndPath( ctx context.Context, thread core.ID, - key, pth string, + key string, identity did.Token, + pth string, ) (*collection.Bucket, path.Path, error) { instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) if err != nil { @@ -175,6 +177,7 @@ func (b *Buckets) getBucketAndPath( return instance, bpth, err } +// getBucketPath concatenates the bucket root path with path. func getBucketPath(bucket *collection.Bucket, pth string) (path.Path, error) { bpth := path.New(gopath.Join(bucket.Path, pth)) if err := bpth.IsValid(); err != nil { diff --git a/pinning/openapi/.openapi-generator-ignore b/pinning/openapi/.openapi-generator-ignore new file mode 100644 index 0000000..7484ee5 --- /dev/null +++ b/pinning/openapi/.openapi-generator-ignore @@ -0,0 +1,23 @@ +# OpenAPI Generator Ignore +# Generated by openapi-generator https://github.com/openapitools/openapi-generator + +# Use this file to prevent files from being overwritten by the generator. +# The patterns follow closely to .gitignore or .dockerignore. + +# As an example, the C# client generator defines ApiClient.cs. +# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: +#ApiClient.cs + +# You can match any string of characters against a directory, file or extension with a single asterisk (*): +#foo/*/qux +# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux + +# You can recursively match patterns against a directory, file or extension with a double asterisk (**): +#foo/**/qux +# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux + +# You can also negate patterns with an exclamation (!). +# For example, you can ignore all files in a docs folder with the file extension .md: +#docs/*.md +# Then explicitly reverse the ignore rule for a single file: +#!docs/README.md diff --git a/pinning/openapi/.openapi-generator/FILES b/pinning/openapi/.openapi-generator/FILES new file mode 100644 index 0000000..6b37b47 --- /dev/null +++ b/pinning/openapi/.openapi-generator/FILES @@ -0,0 +1,14 @@ +.openapi-generator-ignore +Dockerfile +api/openapi.yaml +go/README.md +go/api_pins.go +go/model_failure.go +go/model_failure_error.go +go/model_pin.go +go/model_pin_results.go +go/model_pin_status.go +go/model_status.go +go/model_text_matching_strategy.go +go/routers.go +main.go diff --git a/pinning/openapi/.openapi-generator/VERSION b/pinning/openapi/.openapi-generator/VERSION new file mode 100644 index 0000000..32f3eaa --- /dev/null +++ b/pinning/openapi/.openapi-generator/VERSION @@ -0,0 +1 @@ +5.0.1 \ No newline at end of file diff --git a/pinning/openapi/api/openapi.yaml b/pinning/openapi/api/openapi.yaml new file mode 100644 index 0000000..e610283 --- /dev/null +++ b/pinning/openapi/api/openapi.yaml @@ -0,0 +1,1067 @@ +openapi: 3.0.0 +info: + description: |2+ + + + ## About this spec + The IPFS Pinning Service API is intended to be an implementation-agnostic API: + - For use and implementation by pinning service providers + - For use in client mode by IPFS nodes and GUI-based applications + + > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** + + # Schemas + This section describes the most important object types and conventions. + + A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). + + ## Identifiers + ### cid + [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. + ### requestid + Unique identifier of a pin request. + + When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. + + Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. + + ## Objects + ### Pin object + + ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) + + The `Pin` object is a representation of a pin request. + + It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. + + ### Pin status response + + ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) + + The `PinStatus` object is a representation of the current state of a pinning operation. + It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. + + # The pin lifecycle + + ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) + + ## Creating a new pin object + The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: + - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future + - `status` in `PinStatus` indicates the current state of a pin + + ## Checking status of in-progress pinning + `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. + + In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. + + ## Replacing an existing pin object + The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. + + ## Removing a pin object + A pin object can be removed via `DELETE /pins/{requestid}`. + + + # Provider hints + A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. + + The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. + + This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. + + **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. + + **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. + + # Custom metadata + Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. + ## Pin metadata + String keys and values passed in `Pin.meta` are persisted with the pin object. + + Potential uses: + - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={"app_id":}` + - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) + + Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. + + ## Pin status info + Additional `PinStatus.info` can be returned by pinning service. + + Potential uses: + - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) + - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead + - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) + - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire + + # Pagination and filtering + Pin objects can be listed by executing `GET /pins` with optional parameters: + + - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. + - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). + - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. + - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. + - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. + + > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + + title: IPFS Pinning Service API + version: 1.0.0 + x-logo: + url: https://bafybeidehxarrk54mkgyl5yxbgjzqilp6tkaz2or36jhq24n3rdtuven54.ipfs.dweb.link/?filename=ipfs-pinning-service.svg +servers: +- url: https://pinning-service.example.com +security: +- accessToken: [] +paths: + /pins: + get: + description: List all the pin objects, matching optional filters; when no filter + is provided, only successful pins are returned + parameters: + - description: Return pin objects responsible for pinning the specified CID(s); + be aware that using longer hash functions introduces further constraints + on the number of CIDs that will fit under the limit of 2000 characters per + URL in browser contexts + example: + - Qm1 + - Qm2 + - bafy3 + explode: false + in: query + name: cid + required: false + schema: + items: + type: string + maxItems: 10 + minItems: 1 + type: array + uniqueItems: true + style: form + - description: Return pin objects with specified name (by default a case-sensitive, + exact match) + example: PreciousData.pdf + explode: true + in: query + name: name + required: false + schema: + maxLength: 255 + type: string + style: form + - description: Customize the text matching strategy applied when name filter + is present + example: exact + explode: true + in: query + name: match + required: false + schema: + $ref: '#/components/schemas/TextMatchingStrategy' + style: form + - description: Return pin objects for pins with the specified status + example: + - queued + - pinning + explode: false + in: query + name: status + required: false + schema: + items: + $ref: '#/components/schemas/Status' + minItems: 1 + type: array + uniqueItems: true + style: form + - description: Return results created (queued) before provided timestamp + example: 2020-07-27T17:32:28Z + explode: true + in: query + name: before + required: false + schema: + format: date-time + type: string + style: form + - description: Return results created (queued) after provided timestamp + example: 2020-07-27T17:32:28Z + explode: true + in: query + name: after + required: false + schema: + format: date-time + type: string + style: form + - description: Max records to return + explode: true + in: query + name: limit + required: false + schema: + default: 10 + format: int32 + maximum: 1000 + minimum: 1 + type: integer + style: form + - content: + application/json: + schema: + $ref: '#/components/schemas/PinMeta' + description: Return pin objects that match specified metadata + explode: true + in: query + name: meta + required: false + style: form + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/PinResults' + description: Successful response (PinResults object) + "400": + content: + application/json: + examples: + BadRequestExample: + $ref: '#/components/examples/BadRequestExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Bad request) + "401": + content: + application/json: + examples: + UnauthorizedExample: + $ref: '#/components/examples/UnauthorizedExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unauthorized; access token is missing or invalid) + "404": + content: + application/json: + examples: + NotFoundExample: + $ref: '#/components/examples/NotFoundExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (The specified resource was not found) + "409": + content: + application/json: + examples: + InsufficientFundsExample: + $ref: '#/components/examples/InsufficientFundsExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Insufficient funds) + "4XX": + content: + application/json: + examples: + CustomServiceErrorExample: + $ref: '#/components/examples/CustomServiceErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Custom service error) + "5XX": + content: + application/json: + examples: + InternalServerErrorExample: + $ref: '#/components/examples/InternalServerErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unexpected internal server error) + summary: List pin objects + tags: + - pins + post: + description: Add a new pin object for the current access token + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Pin' + required: true + responses: + "202": + content: + application/json: + schema: + $ref: '#/components/schemas/PinStatus' + description: Successful response (PinStatus object) + "400": + content: + application/json: + examples: + BadRequestExample: + $ref: '#/components/examples/BadRequestExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Bad request) + "401": + content: + application/json: + examples: + UnauthorizedExample: + $ref: '#/components/examples/UnauthorizedExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unauthorized; access token is missing or invalid) + "404": + content: + application/json: + examples: + NotFoundExample: + $ref: '#/components/examples/NotFoundExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (The specified resource was not found) + "409": + content: + application/json: + examples: + InsufficientFundsExample: + $ref: '#/components/examples/InsufficientFundsExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Insufficient funds) + "4XX": + content: + application/json: + examples: + CustomServiceErrorExample: + $ref: '#/components/examples/CustomServiceErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Custom service error) + "5XX": + content: + application/json: + examples: + InternalServerErrorExample: + $ref: '#/components/examples/InternalServerErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unexpected internal server error) + summary: Add pin object + tags: + - pins + /pins/{requestid}: + delete: + description: Remove a pin object + parameters: + - explode: false + in: path + name: requestid + required: true + schema: + type: string + style: simple + responses: + "202": + description: Successful response (no body, pin removed) + "400": + content: + application/json: + examples: + BadRequestExample: + $ref: '#/components/examples/BadRequestExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Bad request) + "401": + content: + application/json: + examples: + UnauthorizedExample: + $ref: '#/components/examples/UnauthorizedExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unauthorized; access token is missing or invalid) + "404": + content: + application/json: + examples: + NotFoundExample: + $ref: '#/components/examples/NotFoundExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (The specified resource was not found) + "409": + content: + application/json: + examples: + InsufficientFundsExample: + $ref: '#/components/examples/InsufficientFundsExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Insufficient funds) + "4XX": + content: + application/json: + examples: + CustomServiceErrorExample: + $ref: '#/components/examples/CustomServiceErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Custom service error) + "5XX": + content: + application/json: + examples: + InternalServerErrorExample: + $ref: '#/components/examples/InternalServerErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unexpected internal server error) + summary: Remove pin object + tags: + - pins + get: + description: Get a pin object and its status + parameters: + - explode: false + in: path + name: requestid + required: true + schema: + type: string + style: simple + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/PinStatus' + description: Successful response (PinStatus object) + "400": + content: + application/json: + examples: + BadRequestExample: + $ref: '#/components/examples/BadRequestExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Bad request) + "401": + content: + application/json: + examples: + UnauthorizedExample: + $ref: '#/components/examples/UnauthorizedExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unauthorized; access token is missing or invalid) + "404": + content: + application/json: + examples: + NotFoundExample: + $ref: '#/components/examples/NotFoundExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (The specified resource was not found) + "409": + content: + application/json: + examples: + InsufficientFundsExample: + $ref: '#/components/examples/InsufficientFundsExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Insufficient funds) + "4XX": + content: + application/json: + examples: + CustomServiceErrorExample: + $ref: '#/components/examples/CustomServiceErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Custom service error) + "5XX": + content: + application/json: + examples: + InternalServerErrorExample: + $ref: '#/components/examples/InternalServerErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unexpected internal server error) + summary: Get pin object + tags: + - pins + post: + description: Replace an existing pin object (shortcut for executing remove and + add operations in one step to avoid unnecessary garbage collection of blocks + present in both recursive pins) + parameters: + - explode: false + in: path + name: requestid + required: true + schema: + type: string + style: simple + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Pin' + required: true + responses: + "202": + content: + application/json: + schema: + $ref: '#/components/schemas/PinStatus' + description: Successful response (PinStatus object) + "400": + content: + application/json: + examples: + BadRequestExample: + $ref: '#/components/examples/BadRequestExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Bad request) + "401": + content: + application/json: + examples: + UnauthorizedExample: + $ref: '#/components/examples/UnauthorizedExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unauthorized; access token is missing or invalid) + "404": + content: + application/json: + examples: + NotFoundExample: + $ref: '#/components/examples/NotFoundExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (The specified resource was not found) + "409": + content: + application/json: + examples: + InsufficientFundsExample: + $ref: '#/components/examples/InsufficientFundsExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Insufficient funds) + "4XX": + content: + application/json: + examples: + CustomServiceErrorExample: + $ref: '#/components/examples/CustomServiceErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Custom service error) + "5XX": + content: + application/json: + examples: + InternalServerErrorExample: + $ref: '#/components/examples/InternalServerErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unexpected internal server error) + summary: Replace pin object + tags: + - pins +components: + examples: + BadRequestExample: + summary: A sample response to a bad request; reason will differ + value: + error: + reason: BAD_REQUEST + details: Explanation for humans with more details + UnauthorizedExample: + summary: Response to an unauthorized request + value: + error: + reason: UNAUTHORIZED + details: Access token is missing or invalid + NotFoundExample: + summary: Response to a request for a resource that does not exist + value: + error: + reason: NOT_FOUND + details: The specified resource was not found + InsufficientFundsExample: + summary: Response when access token run out of funds + value: + error: + reason: INSUFFICIENT_FUNDS + details: Unable to process request due to the lack of funds + CustomServiceErrorExample: + summary: Response when a custom error occured + value: + error: + reason: CUSTOM_ERROR_CODE_FOR_MACHINES + details: Optional explanation for humans with more details + InternalServerErrorExample: + summary: Response when unexpected error occured + value: + error: + reason: INTERNAL_SERVER_ERROR + details: Explanation for humans with more details + parameters: + before: + description: Return results created (queued) before provided timestamp + example: 2020-07-27T17:32:28Z + explode: true + in: query + name: before + required: false + schema: + format: date-time + type: string + style: form + after: + description: Return results created (queued) after provided timestamp + example: 2020-07-27T17:32:28Z + explode: true + in: query + name: after + required: false + schema: + format: date-time + type: string + style: form + limit: + description: Max records to return + explode: true + in: query + name: limit + required: false + schema: + default: 10 + format: int32 + maximum: 1000 + minimum: 1 + type: integer + style: form + cid: + description: Return pin objects responsible for pinning the specified CID(s); + be aware that using longer hash functions introduces further constraints on + the number of CIDs that will fit under the limit of 2000 characters per URL in + browser contexts + example: + - Qm1 + - Qm2 + - bafy3 + explode: false + in: query + name: cid + required: false + schema: + items: + type: string + maxItems: 10 + minItems: 1 + type: array + uniqueItems: true + style: form + name: + description: Return pin objects with specified name (by default a case-sensitive, + exact match) + example: PreciousData.pdf + explode: true + in: query + name: name + required: false + schema: + maxLength: 255 + type: string + style: form + match: + description: Customize the text matching strategy applied when name filter is + present + example: exact + explode: true + in: query + name: match + required: false + schema: + $ref: '#/components/schemas/TextMatchingStrategy' + style: form + status: + description: Return pin objects for pins with the specified status + example: + - queued + - pinning + explode: false + in: query + name: status + required: false + schema: + items: + $ref: '#/components/schemas/Status' + minItems: 1 + type: array + uniqueItems: true + style: form + meta: + content: + application/json: + schema: + $ref: '#/components/schemas/PinMeta' + description: Return pin objects that match specified metadata + explode: true + in: query + name: meta + required: false + style: form + responses: + BadRequest: + content: + application/json: + examples: + BadRequestExample: + $ref: '#/components/examples/BadRequestExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Bad request) + Unauthorized: + content: + application/json: + examples: + UnauthorizedExample: + $ref: '#/components/examples/UnauthorizedExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unauthorized; access token is missing or invalid) + NotFound: + content: + application/json: + examples: + NotFoundExample: + $ref: '#/components/examples/NotFoundExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (The specified resource was not found) + InsufficientFunds: + content: + application/json: + examples: + InsufficientFundsExample: + $ref: '#/components/examples/InsufficientFundsExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Insufficient funds) + CustomServiceError: + content: + application/json: + examples: + CustomServiceErrorExample: + $ref: '#/components/examples/CustomServiceErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Custom service error) + InternalServerError: + content: + application/json: + examples: + InternalServerErrorExample: + $ref: '#/components/examples/InternalServerErrorExample' + schema: + $ref: '#/components/schemas/Failure' + description: Error response (Unexpected internal server error) + schemas: + PinResults: + description: Response used for listing pin objects matching request + example: + count: 1 + results: + - pin: + meta: + app_id: 99986338-1113-4706-8302-4420da6158aa + name: PreciousData.pdf + origins: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + cid: QmCIDToBePinned + requestid: UniqueIdOfPinRequest + created: 2020-07-27T17:32:28Z + delegates: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + info: + status_details: 'Queue position: 7 of 9' + - pin: + meta: + app_id: 99986338-1113-4706-8302-4420da6158aa + name: PreciousData.pdf + origins: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + cid: QmCIDToBePinned + requestid: UniqueIdOfPinRequest + created: 2020-07-27T17:32:28Z + delegates: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + info: + status_details: 'Queue position: 7 of 9' + - pin: + meta: + app_id: 99986338-1113-4706-8302-4420da6158aa + name: PreciousData.pdf + origins: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + cid: QmCIDToBePinned + requestid: UniqueIdOfPinRequest + created: 2020-07-27T17:32:28Z + delegates: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + info: + status_details: 'Queue position: 7 of 9' + - pin: + meta: + app_id: 99986338-1113-4706-8302-4420da6158aa + name: PreciousData.pdf + origins: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + cid: QmCIDToBePinned + requestid: UniqueIdOfPinRequest + created: 2020-07-27T17:32:28Z + delegates: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + info: + status_details: 'Queue position: 7 of 9' + - pin: + meta: + app_id: 99986338-1113-4706-8302-4420da6158aa + name: PreciousData.pdf + origins: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + cid: QmCIDToBePinned + requestid: UniqueIdOfPinRequest + created: 2020-07-27T17:32:28Z + delegates: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + info: + status_details: 'Queue position: 7 of 9' + properties: + count: + description: The total number of pin objects that exist for passed query + filters + example: 1 + format: int32 + minimum: 0 + type: integer + results: + description: An array of PinStatus results + items: + $ref: '#/components/schemas/PinStatus' + maxItems: 1000 + minItems: 0 + type: array + uniqueItems: true + required: + - count + - results + type: object + PinStatus: + description: Pin object with status + example: + pin: + meta: + app_id: 99986338-1113-4706-8302-4420da6158aa + name: PreciousData.pdf + origins: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + cid: QmCIDToBePinned + requestid: UniqueIdOfPinRequest + created: 2020-07-27T17:32:28Z + delegates: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + info: + status_details: 'Queue position: 7 of 9' + properties: + requestid: + description: Globally unique identifier of the pin request; can be used + to check the status of ongoing pinning, or pin removal + example: UniqueIdOfPinRequest + type: string + status: + $ref: '#/components/schemas/Status' + created: + description: Immutable timestamp indicating when a pin request entered a + pinning service; can be used for filtering results and pagination + example: 2020-07-27T17:32:28Z + format: date-time + type: string + pin: + $ref: '#/components/schemas/Pin' + delegates: + description: List of multiaddrs designated by pinning service for transferring + any new data from external peers + example: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + items: + type: string + maxItems: 20 + minItems: 1 + type: array + uniqueItems: true + info: + additionalProperties: + maxProperties: 1000 + minProperties: 0 + type: string + description: Optional info for PinStatus response + example: + status_details: 'Queue position: 7 of 9' + type: object + required: + - created + - delegates + - pin + - requestid + - status + type: object + Pin: + description: Pin object + example: + meta: + app_id: 99986338-1113-4706-8302-4420da6158aa + name: PreciousData.pdf + origins: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + cid: QmCIDToBePinned + properties: + cid: + description: Content Identifier (CID) to be pinned recursively + example: QmCIDToBePinned + type: string + name: + description: Optional name for pinned data; can be used for lookups later + example: PreciousData.pdf + maxLength: 255 + type: string + origins: + description: Optional list of multiaddrs known to provide the data + example: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + items: + type: string + maxItems: 20 + minItems: 0 + type: array + uniqueItems: true + meta: + additionalProperties: + maxProperties: 1000 + minProperties: 0 + type: string + description: Optional metadata for pin object + example: + app_id: 99986338-1113-4706-8302-4420da6158aa + type: object + required: + - cid + type: object + Status: + description: Status a pin object can have at a pinning service + enum: + - queued + - pinning + - pinned + - failed + type: string + Delegates: + description: List of multiaddrs designated by pinning service for transferring + any new data from external peers + example: + - /ip4/203.0.113.1/tcp/4001/p2p/QmServicePeerId + items: + type: string + maxItems: 20 + minItems: 1 + type: array + uniqueItems: true + Origins: + description: Optional list of multiaddrs known to provide the data + example: + - /ip4/203.0.113.142/tcp/4001/p2p/QmSourcePeerId + - /ip4/203.0.113.114/udp/4001/quic/p2p/QmSourcePeerId + items: + type: string + maxItems: 20 + minItems: 0 + type: array + uniqueItems: true + PinMeta: + additionalProperties: + maxProperties: 1000 + minProperties: 0 + type: string + description: Optional metadata for pin object + example: + app_id: 99986338-1113-4706-8302-4420da6158aa + type: object + StatusInfo: + additionalProperties: + maxProperties: 1000 + minProperties: 0 + type: string + description: Optional info for PinStatus response + example: + status_details: 'Queue position: 7 of 9' + type: object + TextMatchingStrategy: + default: exact + description: Alternative text matching strategy + enum: + - exact + - iexact + - partial + - ipartial + type: string + Failure: + description: Response for a failed request + properties: + error: + $ref: '#/components/schemas/Failure_error' + required: + - error + type: object + Failure_error: + properties: + reason: + description: Mandatory string identifying the type of error + example: ERROR_CODE_FOR_MACHINES + type: string + details: + description: Optional, longer description of the error; may include UUID + of transaction for support, links to documentation etc + example: Optional explanation for humans with more details + type: string + required: + - reason + type: object + securitySchemes: + accessToken: + description: " An opaque token is required to be sent with each request in the\ + \ HTTP header:\n- `Authorization: Bearer `\n\nThe `access-token`\ + \ should be generated per device, and the user should have the ability to\ + \ revoke each token separately. " + scheme: bearer + type: http diff --git a/pinning/openapi/go/README.md b/pinning/openapi/go/README.md new file mode 100644 index 0000000..1bdb972 --- /dev/null +++ b/pinning/openapi/go/README.md @@ -0,0 +1,144 @@ +# Go API Server for openapi + + + +## About this spec +The IPFS Pinning Service API is intended to be an implementation-agnostic API: +- For use and implementation by pinning service providers +- For use in client mode by IPFS nodes and GUI-based applications + +> **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** + +# Schemas +This section describes the most important object types and conventions. + +A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). + +## Identifiers +### cid +[Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. +### requestid +Unique identifier of a pin request. + +When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. + +Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. + +## Objects +### Pin object + +![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) + +The `Pin` object is a representation of a pin request. + +It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. + +### Pin status response + +![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) + +The `PinStatus` object is a representation of the current state of a pinning operation. +It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. + +# The pin lifecycle + +![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) + +## Creating a new pin object +The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: +- `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future +- `status` in `PinStatus` indicates the current state of a pin + +## Checking status of in-progress pinning +`status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. + +In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. + +## Replacing an existing pin object +The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. + +## Removing a pin object +A pin object can be removed via `DELETE /pins/{requestid}`. + + +# Provider hints +A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. + +The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. + +This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. + +**NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. + +**NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. + +# Custom metadata +Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. +## Pin metadata +String keys and values passed in `Pin.meta` are persisted with the pin object. + +Potential uses: +- `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` +- `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) + +Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. + +## Pin status info +Additional `PinStatus.info` can be returned by pinning service. + +Potential uses: +- `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) +- `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead +- `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) +- `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire + +# Pagination and filtering +Pin objects can be listed by executing `GET /pins` with optional parameters: + +- When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. +- The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). +- If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. +- To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. +- Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. + +> **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + + + +## Overview +This server was generated by the [openapi-generator] +(https://openapi-generator.tech) project. +By using the [OpenAPI-Spec](https://github.com/OAI/OpenAPI-Specification) from a remote server, you can easily generate a server stub. +- + +To see how to make this your own, look here: + +[README](https://openapi-generator.tech) + +- API version: 1.0.0 +- Build date: 2021-03-17T15:36:11.936787-07:00[America/Los_Angeles] + +### Running the server + +To run the server, follow these simple steps: + +``` +go run main.go +``` + +To run the server in a docker container +``` +docker build --network=host -t openapi . +``` + +Once the image is built, just run +``` +docker run --rm -it openapi +``` + +### Known Issue + +Endpoints sharing a common path may result in issues. For example, `/v2/pet/findByTags` and `/v2/pet/:petId` will result in an issue with the Gin framework. For more information about this known limitation, please refer to [gin-gonic/gin#388](https://github.com/gin-gonic/gin/issues/388) for more information. + +A workaround is to manually update the path and handler. Please refer to [gin-gonic/gin/issues/205#issuecomment-296155497](https://github.com/gin-gonic/gin/issues/205#issuecomment-296155497) for more information. + diff --git a/pinning/openapi/go/model_failure.go b/pinning/openapi/go/model_failure.go new file mode 100644 index 0000000..b0cd62e --- /dev/null +++ b/pinning/openapi/go/model_failure.go @@ -0,0 +1,16 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. # The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ## Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ## Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ## Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ## Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. # Provider hints A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. # Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ## Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ## Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 1.0.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +// Failure - Response for a failed request +type Failure struct { + + Error FailureError `json:"error"` +} diff --git a/pinning/openapi/go/model_failure_error.go b/pinning/openapi/go/model_failure_error.go new file mode 100644 index 0000000..bcc15f9 --- /dev/null +++ b/pinning/openapi/go/model_failure_error.go @@ -0,0 +1,19 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. # The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ## Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ## Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ## Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ## Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. # Provider hints A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. # Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ## Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ## Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 1.0.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +type FailureError struct { + + // Mandatory string identifying the type of error + Reason string `json:"reason"` + + // Optional, longer description of the error; may include UUID of transaction for support, links to documentation etc + Details string `json:"details,omitempty"` +} diff --git a/pinning/openapi/go/model_pin.go b/pinning/openapi/go/model_pin.go new file mode 100644 index 0000000..f5a8928 --- /dev/null +++ b/pinning/openapi/go/model_pin.go @@ -0,0 +1,26 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. # The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ## Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ## Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ## Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ## Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. # Provider hints A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. # Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ## Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ## Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 1.0.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +// Pin - Pin object +type Pin struct { + + // Content Identifier (CID) to be pinned recursively + Cid string `json:"cid"` + + // Optional name for pinned data; can be used for lookups later + Name string `json:"name,omitempty"` + + // Optional list of multiaddrs known to provide the data + Origins []string `json:"origins,omitempty"` + + // Optional metadata for pin object + Meta map[string]string `json:"meta,omitempty"` +} diff --git a/pinning/openapi/go/model_pin_results.go b/pinning/openapi/go/model_pin_results.go new file mode 100644 index 0000000..df5f53b --- /dev/null +++ b/pinning/openapi/go/model_pin_results.go @@ -0,0 +1,20 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. # The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ## Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ## Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ## Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ## Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. # Provider hints A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. # Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ## Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ## Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 1.0.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +// PinResults - Response used for listing pin objects matching request +type PinResults struct { + + // The total number of pin objects that exist for passed query filters + Count int32 `json:"count"` + + // An array of PinStatus results + Results []PinStatus `json:"results"` +} diff --git a/pinning/openapi/go/model_pin_status.go b/pinning/openapi/go/model_pin_status.go new file mode 100644 index 0000000..fd6d0ca --- /dev/null +++ b/pinning/openapi/go/model_pin_status.go @@ -0,0 +1,34 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. # The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ## Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ## Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ## Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ## Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. # Provider hints A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. # Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ## Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ## Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 1.0.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi + +import ( + "time" +) + +// PinStatus - Pin object with status +type PinStatus struct { + + // Globally unique identifier of the pin request; can be used to check the status of ongoing pinning, or pin removal + Requestid string `json:"requestid"` + + Status Status `json:"status"` + + // Immutable timestamp indicating when a pin request entered a pinning service; can be used for filtering results and pagination + Created time.Time `json:"created"` + + Pin Pin `json:"pin"` + + // List of multiaddrs designated by pinning service for transferring any new data from external peers + Delegates []string `json:"delegates"` + + // Optional info for PinStatus response + Info map[string]string `json:"info,omitempty"` +} diff --git a/pinning/openapi/go/model_query.go b/pinning/openapi/go/model_query.go new file mode 100644 index 0000000..5595e76 --- /dev/null +++ b/pinning/openapi/go/model_query.go @@ -0,0 +1,31 @@ +package openapi + +import ( + "time" +) + +// Query represents Pin query parameters. +// This is derived from the openapi spec using https://github.com/deepmap/oapi-codegen, not +// https://github.com/OpenAPITools/openapi-generator, which doens't output anything for the +// listPins query body. +type Query struct { + // Cid can be used to filter by one or more Pin Cids. + Cid []string `form:"cid" json:"cid,omitempty"` + // Name can be used to filer by Pin name (by default case-sensitive, exact match). + Name string `form:"name" json:"name,omitempty"` + // Match can be used to customize the text matching strategy applied when Name is present. + Match string `form:"match" json:"match,omitempty"` + // Status can be used to filter by Pin status. + Status string `form:"status" json:"status,omitempty"` + // Before can by used to filter by before creation (queued) time. + Before time.Time `form:"before" json:"before,omitempty"` + // After can by used to filter by after creation (queued) time. + After time.Time `form:"after" json:"after,omitempty"` + // Limit specifies the max number of Pins to return. + Limit int32 `form:"limit" json:"limit,omitempty"` +} + +// QueryMeta can be used to filter results by Pin metadata. +// This was pulled out of the openapi generated Query above because gin is not able to +// auto parse the map string value sent by the generic pinning client, i.e., "meta=map[foo:one bar:two]". +type QueryMeta map[string]string diff --git a/pinning/openapi/go/model_status.go b/pinning/openapi/go/model_status.go new file mode 100644 index 0000000..86543eb --- /dev/null +++ b/pinning/openapi/go/model_status.go @@ -0,0 +1,20 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. # The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ## Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ## Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ## Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ## Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. # Provider hints A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. # Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ## Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ## Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 1.0.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi +// Status : Status a pin object can have at a pinning service +type Status string + +// List of Status +const ( + QUEUED Status = "queued" + PINNING Status = "pinning" + PINNED Status = "pinned" + FAILED Status = "failed" +) diff --git a/pinning/openapi/go/model_text_matching_strategy.go b/pinning/openapi/go/model_text_matching_strategy.go new file mode 100644 index 0000000..348ac9d --- /dev/null +++ b/pinning/openapi/go/model_text_matching_strategy.go @@ -0,0 +1,20 @@ +/* + * IPFS Pinning Service API + * + * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object ![pin object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pin.png) The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response ![pin status response object](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/pinstatus.png) The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. # The pin lifecycle ![pinning service objects and lifecycle](https://bafybeideck2fchyxna4wqwc2mo67yriokehw3yujboc5redjdaajrk2fjq.ipfs.dweb.link/lifecycle.png) ## Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ## Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ## Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ## Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. # Provider hints A pinning service will use the DHT and other discovery methods to locate pinned content; however, it is a good practice to provide additional provider hints to speed up the discovery phase and start the transfer immediately, especially if a client has the data in their own datastore or already knows of other providers. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then attempt to connect to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. At the same time, a pinning service will try to connect to multiaddrs provided by the client in `Pin.origins`. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and mutual direct dial between a client and a service works around peer routing issues in restrictive network topologies, such as NATs, firewalls, etc. **NOTE:** Connections to multiaddrs in `origins` and `delegates` arrays should be attempted in best-effort fashion, and dial failure should not fail the pinning operation. When unable to act on explicit provider hints, DHT and other discovery methods should be used as a fallback by a pinning service. **NOTE:** All multiaddrs MUST end with `/p2p/{peerID}` and SHOULD be fully resolved and confirmed to be dialable from the public internet. Avoid sending addresses from local networks. # Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ## Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ## Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this. + * + * API version: 1.0.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi +// TextMatchingStrategy : Alternative text matching strategy +type TextMatchingStrategy string + +// List of TextMatchingStrategy +const ( + EXACT TextMatchingStrategy = "exact" + IEXACT TextMatchingStrategy = "iexact" + PARTIAL TextMatchingStrategy = "partial" + IPARTIAL TextMatchingStrategy = "ipartial" +) diff --git a/pinning/queue/queue.go b/pinning/queue/queue.go new file mode 100644 index 0000000..27ec580 --- /dev/null +++ b/pinning/queue/queue.go @@ -0,0 +1,690 @@ +package queue + +// @todo: Handle reload in-progress pins after shutdown +// @todo: Delete did.Token from request after success/fail +// @todo: Batch jobs by key, then handler can directly fetch cids with IPFS and use PushPaths to save bucket writes + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/gob" + "errors" + "fmt" + "strings" + "sync" + "time" + + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + logging "github.com/ipfs/go-log/v2" + "github.com/oklog/ulid/v2" + openapi "github.com/textileio/go-buckets/pinning/openapi/go" + dsextensions "github.com/textileio/go-datastore-extensions" + "github.com/textileio/go-threads/core/did" + core "github.com/textileio/go-threads/core/thread" + kt "github.com/textileio/go-threads/db/keytransform" +) + +var ( + log = logging.Logger("buckets/ps-queue") + + // StartDelay is the time delay before the queue will process queued request on start. + StartDelay = time.Second * 10 + + // MaxConcurrency is the maximum number of requests that will be handled concurrently. + MaxConcurrency = 100 + + // ErrNotFound indicates the requested request was not found. + ErrNotFound = errors.New("request not found") + + // ErrInProgress indicates the request is in progress and cannot be altered. + ErrInProgress = errors.New("request in progress") + + // dsQueuePrefix is the prefix for global time-ordered keys used internally for processing. + // Structure: /queue/ + dsQueuePrefix = ds.NewKey("/queue") + + // dsGroupPrefix is the prefix for grouped time-ordered keys used to list requests. + // Structure: /group// + dsGroupPrefix = ds.NewKey("/group") +) + +const ( + // defaultListLimit is the default request list page size. + defaultListLimit = 10 + // maxListLimit is the max request list page size. + maxListLimit = 1000 +) + +// Request is a wrapper for openapi.PinStatus that is persisted to the datastore. +type Request struct { + openapi.PinStatus + + Thread core.ID + Key string + Identity did.Token + + // Replace indicates openapi.PinStatus.Pin.Cid is marked for replacement. + Replace bool + // Remove indicates the request is marked for removal. + Remove bool +} + +// RequestParams are used to create a new request. +type RequestParams struct { + openapi.Pin + + // Time the request was received (used for openapi.PinStatus.Created). + Time time.Time + + Thread core.ID + Key string + Identity did.Token +} + +// Query is used to query for requests (a more typed version of openapi.Query). +type Query struct { + Cids []string + Name string + Match openapi.TextMatchingStrategy + Statuses []openapi.Status + Before time.Time + After time.Time + Limit int + Meta map[string]string +} + +func (q Query) setDefaults() Query { + if q.Limit == -1 { + q.Limit = maxListLimit + } else if q.Limit <= 0 { + q.Limit = defaultListLimit + } else if q.Limit > maxListLimit { + q.Limit = maxListLimit + } + if q.Meta == nil { + q.Meta = make(map[string]string) + } + if len(q.Match) == 0 { + q.Match = openapi.EXACT + } + return q +} + +// Handler is called when a request moves from "queued" to "pinning". +// This separates the queue's job from the actual handling of a request, making the queue logic easier to test. +type Handler func(ctx context.Context, request Request) error + +// Queue is a persistent worker-based task queue. +type Queue struct { + store kt.TxnDatastoreExtended + + handler Handler + jobCh chan Request + doneCh chan struct{} + entropy *ulid.MonotonicEntropy + + ctx context.Context + cancel context.CancelFunc + + lk sync.Mutex +} + +// NewQueue returns a new Queue using handler to process requests. +func NewQueue(store kt.TxnDatastoreExtended, handler Handler) (*Queue, error) { + ctx, cancel := context.WithCancel(context.Background()) + q := &Queue{ + store: store, + handler: handler, + jobCh: make(chan Request, MaxConcurrency), + doneCh: make(chan struct{}, MaxConcurrency), + ctx: ctx, + cancel: cancel, + } + + // Create queue workers + for i := 0; i < MaxConcurrency; i++ { + go q.worker(i + 1) + } + + go q.start() + return q, nil +} + +// Close the queue and cancel active "pinning" requests. +func (q *Queue) Close() error { + q.cancel() + return nil +} + +// NewID returns new monotonically increasing request ids. +func (q *Queue) NewID(t time.Time) (string, error) { + q.lk.Lock() // entropy is not safe for concurrent use + + if q.entropy == nil { + q.entropy = ulid.Monotonic(rand.Reader, 0) + } + id, err := ulid.New(ulid.Timestamp(t.UTC()), q.entropy) + if errors.Is(err, ulid.ErrMonotonicOverflow) { + q.entropy = nil + q.lk.Unlock() + return q.NewID(t) + } else if err != nil { + q.lk.Unlock() + return "", fmt.Errorf("generating requestid: %v", err) + } + q.lk.Unlock() + return strings.ToLower(id.String()), nil +} + +// cidFilter is used to query for one or more requests with a particular openapi.PinStatus.Pin.Cid. +type cidFilter struct { + ok []string +} + +func (f *cidFilter) Filter(e dsq.Entry) bool { + r, err := decode(e.Value) + if err != nil { + log.Errorf("error decoding entry: %v", err) + return false + } + for _, ok := range f.ok { + if r.Pin.Cid == ok { + return true + } + } + return false +} + +// nameFilter is used to query for one or more requests with a particular openapi.PinStatus.Pin.Name. +type nameFilter struct { + name string + match openapi.TextMatchingStrategy +} + +func (f *nameFilter) Filter(e dsq.Entry) bool { + r, err := decode(e.Value) + if err != nil { + log.Errorf("error decoding entry: %v", err) + return false + } + switch f.match { + case openapi.EXACT: + return r.Pin.Name == f.name + case openapi.IEXACT: + return strings.ToLower(r.Pin.Name) == strings.ToLower(f.name) + case openapi.PARTIAL: + return strings.Contains(r.Pin.Name, f.name) + case openapi.IPARTIAL: + return strings.Contains(strings.ToLower(r.Pin.Name), strings.ToLower(f.name)) + default: + return false + } +} + +// statusFilter is used to query for one or more requests with a particular openapi.PinStatus.Status. +type statusFilter struct { + ok []openapi.Status +} + +func (f *statusFilter) Filter(e dsq.Entry) bool { + r, err := decode(e.Value) + if err != nil { + log.Errorf("error decoding entry: %v", err) + return false + } + for _, ok := range f.ok { + if r.Status == ok { + return true + } + } + return false +} + +// metaFilter is used to query for one or more requests with matching openapi.PinStatus.Pin.Meta. +type metaFilter struct { + meta map[string]string +} + +func (f *metaFilter) Filter(e dsq.Entry) bool { + r, err := decode(e.Value) + if err != nil { + log.Errorf("error decoding entry: %v", err) + return false + } + var match bool +loop: + for fk, fv := range f.meta { + for k, v := range r.Pin.Meta { + if k == fk { + if v == fv { + match = true + continue loop // So far so good, check next filter + } else { + return false // Values don't match, we're done + } + } + } + return false // Key not found, we're done + } + return match +} + +// ListRequests lists requests for a group key by applying a Query. +func (q *Queue) ListRequests(group string, query Query) ([]openapi.PinStatus, error) { + query = query.setDefaults() + if !query.Before.IsZero() && !query.After.IsZero() { + return nil, errors.New("before and after cannot be used together") + } + + var ( + order dsq.Order = dsq.OrderByKeyDescending{} + seek, seekKey string + filters []dsq.Filter + limit = query.Limit + err error + ) + + if !query.After.IsZero() { + order = dsq.OrderByKey{} + seek, err = q.NewID(query.After.Add(time.Millisecond)) + if err != nil { + return nil, fmt.Errorf("getting 'after' id: %v", err) + } + } else { + if query.Before.IsZero() { + seek = strings.ToLower(ulid.MustNew(ulid.MaxTime(), nil).String()) + } else { + seek, err = q.NewID(query.Before.Add(-time.Millisecond)) + if err != nil { + return nil, fmt.Errorf("getting 'before' id: %v", err) + } + } + } + seekKey = getGroupKey(group, seek).String() + + if len(query.Cids) > 0 { + filters = append(filters, &cidFilter{ + ok: query.Cids, + }) + } + if len(query.Name) > 0 { + filters = append(filters, &nameFilter{ + name: query.Name, + match: query.Match, + }) + } + if len(query.Statuses) > 0 { + filters = append(filters, &statusFilter{ + ok: query.Statuses, + }) + } + if len(query.Meta) > 0 { + filters = append(filters, &metaFilter{ + meta: query.Meta, + }) + } + + results, err := q.store.QueryExtended(dsextensions.QueryExt{ + Query: dsq.Query{ + Prefix: dsGroupPrefix.ChildString(group).String(), + Filters: filters, + Orders: []dsq.Order{order}, + Limit: limit, + }, + SeekPrefix: seekKey, + }) + if err != nil { + return nil, fmt.Errorf("querying requests: %v", err) + } + defer results.Close() + + var reqs []openapi.PinStatus + for res := range results.Next() { + if res.Error != nil { + return nil, fmt.Errorf("getting next result: %v", res.Error) + } + r, err := decode(res.Value) + if err != nil { + return nil, fmt.Errorf("decoding request: %v", err) + } + reqs = append(reqs, r.PinStatus) + } + + return reqs, nil +} + +// AddRequest adds a new request to the queue. +// The new request will be handled immediately if workers are not busy. +func (q *Queue) AddRequest(params RequestParams) (*openapi.PinStatus, error) { + id, err := q.NewID(params.Time) + if err != nil { + return nil, fmt.Errorf("creating request id: %v", err) + } + r := Request{ + PinStatus: openapi.PinStatus{ + Requestid: id, + Status: openapi.QUEUED, + Created: params.Time, + Pin: params.Pin, + }, + Thread: params.Thread, + Key: params.Key, + Identity: params.Identity, + } + + if err := q.enqueue(r, true); err != nil { + return nil, fmt.Errorf("enqueueing request: %v", err) + } + return &r.PinStatus, nil +} + +// GetRequest returns a request by group key and id. +func (q *Queue) GetRequest(group, id string) (*openapi.PinStatus, error) { + r, err := q.getRequest(q.store, group, id) + if err != nil { + return nil, err + } + return &r.PinStatus, err +} + +func (q *Queue) getRequest(reader ds.Read, group, id string) (*Request, error) { + val, err := reader.Get(getGroupKey(group, id)) + if errors.Is(err, ds.ErrNotFound) { + return nil, ErrNotFound + } else if err != nil { + return nil, fmt.Errorf("getting group key: %v", err) + } + r, err := decode(val) + if err != nil { + return nil, fmt.Errorf("decoding request: %v", err) + } + return &r, nil +} + +// ReplaceRequest replaces a request's openapi.PinStatus.Pin. +// Note: In-progress ("pinning") requests cannot be replaced. +func (q *Queue) ReplaceRequest(group, id string, pin openapi.Pin) (*openapi.PinStatus, error) { + r, err := q.dequeue(group, id) + if err != nil { + return nil, fmt.Errorf("dequeueing request: %w", err) + } + + // Mark for replacement + r.Replace = true + r.Pin = pin + r.Status = openapi.QUEUED + + // Re-enqueue + if err := q.enqueue(*r, true); err != nil { + return nil, fmt.Errorf("re-enqueueing request: %v", err) + } + return &r.PinStatus, nil +} + +// RemoveRequest removes a request. +// Note: In-progress ("pinning") requests cannot be removed. +func (q *Queue) RemoveRequest(group, id string) error { + r, err := q.dequeue(group, id) + if err != nil { + return fmt.Errorf("dequeueing request: %w", err) + } + + // Mark for removal + r.Remove = true + r.Status = openapi.QUEUED + + // Re-enqueue + if err := q.enqueue(*r, true); err != nil { + return fmt.Errorf("re-enqueueing request: %v", err) + } + return nil +} + +func (q *Queue) enqueue(r Request, isNew bool) error { + // Block while the request is placed in a queue + if isNew { + // Set to "pinning" in case a worker is available now + r.Status = openapi.PINNING + val, err := encode(r) + if err != nil { + return fmt.Errorf("encoding request: %v", err) + } + if err := q.store.Put(getGroupKey(r.Key, r.Requestid), val); err != nil { + return fmt.Errorf("putting group key: %v", err) + } + } else { + // Move the request to the "pinning" queue + if err := q.moveRequest(r, openapi.QUEUED, openapi.PINNING); err != nil { + return fmt.Errorf("updating status (pinning): %v", err) + } + } + + // Unblock the caller by letting the rest happen in the background + go func() { + select { + case q.jobCh <- r: + default: + log.Debugf("workers are busy; queueing %s", r.Requestid) + // Workers are busy, put back in the "queued" queue + if err := q.moveRequest(r, openapi.PINNING, openapi.QUEUED); err != nil { + log.Debugf("error updating status (queued): %v", err) + } + } + }() + return nil +} + +func (q *Queue) dequeue(group, id string) (*Request, error) { + txn, err := q.store.NewTransactionExtended(false) + if err != nil { + return nil, fmt.Errorf("creating txn: %v", err) + } + defer txn.Discard() + + // Check if pinning + r, err := q.getRequest(txn, group, id) + if errors.Is(err, ds.ErrNotFound) { + return nil, ErrNotFound + } else if err != nil { + return nil, fmt.Errorf("getting group key: %v", err) + } else if r.Status == openapi.PINNING { + return nil, ErrInProgress + } + + // Remove queue key + if err := txn.Delete(dsQueuePrefix.ChildString(id)); err != nil { + return nil, fmt.Errorf("putting key: %v", err) + } + + if err := txn.Commit(); err != nil { + return nil, fmt.Errorf("committing txn: %v", err) + } + return r, nil +} + +func (q *Queue) worker(num int) { + for { + select { + case <-q.ctx.Done(): + return + + case r := <-q.jobCh: + if q.ctx.Err() != nil { + return + } + log.Debugf("worker %d got job %s", num, r.Requestid) + + // Handle the request with the handler func + status := openapi.PINNED + if err := q.handler(q.ctx, r); err != nil { + status = openapi.FAILED + log.Debugf("error handling request: %v", err) + } + + if r.Remove { + if err := q.removeRequest(r); err != nil { + log.Debugf("error removing request: %v", err) + } + } else { + // Finalize request by setting status to "pinned" or "failed" + if err := q.moveRequest(r, openapi.PINNING, status); err != nil { + log.Debugf("error updating status (%s): %v", status, err) + } + } + + log.Debugf("worker %d finished job %s", num, r.Requestid) + go func() { + q.doneCh <- struct{}{} + }() + } + } +} + +func (q *Queue) start() { + t := time.NewTimer(StartDelay) + for { + select { + case <-q.ctx.Done(): + t.Stop() + return + case <-t.C: + q.getNext() + case <-q.doneCh: + q.getNext() + } + } +} + +func (q *Queue) getNext() { + queue, err := q.getQueued() + if err != nil { + log.Errorf("listing requests: %v", err) + return + } + if len(queue) > 0 { + log.Debug("enqueueing job: %s", queue[0].Requestid) + } + for _, r := range queue { + if err := q.enqueue(r, false); err != nil { + log.Debugf("error enqueueing request: %v", err) + } + } +} + +func (q *Queue) getQueued() ([]Request, error) { + results, err := q.store.Query(dsq.Query{ + Prefix: dsQueuePrefix.String(), + Orders: []dsq.Order{dsq.OrderByKey{}}, + Limit: 1, + }) + if err != nil { + return nil, fmt.Errorf("querying requests: %v", err) + } + defer results.Close() + + var reqs []Request + for res := range results.Next() { + if res.Error != nil { + return nil, fmt.Errorf("getting next result: %v", res.Error) + } + r, err := decode(res.Value) + if err != nil { + return nil, fmt.Errorf("decoding request: %v", err) + } + reqs = append(reqs, r) + } + return reqs, nil +} + +func (q *Queue) moveRequest(r Request, from, to openapi.Status) error { + txn, err := q.store.NewTransaction(false) + if err != nil { + return fmt.Errorf("creating txn: %v", err) + } + defer txn.Discard() + + // Update status + r.Status = to + + // Reset 'replace' and 'remove' flags if the request is complete + if to == openapi.PINNED || to == openapi.FAILED { + r.Replace = false + r.Remove = false // Only needed if the removal failed + } + + val, err := encode(r) + if err != nil { + return fmt.Errorf("encoding group key: %v", err) + } + + // Handle queue key + if from == openapi.QUEUED { + if err := txn.Delete(dsQueuePrefix.ChildString(r.Requestid)); err != nil { + return fmt.Errorf("deleting key: %v", err) + } + } + if to == openapi.QUEUED { + if err := txn.Put(dsQueuePrefix.ChildString(r.Requestid), val); err != nil { + return fmt.Errorf("putting key: %v", err) + } + } + + // Update group key + if err := txn.Put(getGroupKey(r.Key, r.Requestid), val); err != nil { + return fmt.Errorf("putting group key: %v", err) + } + + if err := txn.Commit(); err != nil { + return fmt.Errorf("committing txn: %v", err) + } + return nil +} + +func (q *Queue) removeRequest(r Request) error { + txn, err := q.store.NewTransaction(false) + if err != nil { + return fmt.Errorf("creating txn: %v", err) + } + defer txn.Discard() + + // Remove queue key + if err := txn.Delete(dsQueuePrefix.ChildString(r.Requestid)); err != nil { + return fmt.Errorf("deleting queue key: %v", err) + } + + // Remove group key + if err := txn.Delete(getGroupKey(r.Key, r.Requestid)); err != nil { + return fmt.Errorf("deleting group key: %v", err) + } + + if err := txn.Commit(); err != nil { + return fmt.Errorf("committing txn: %v", err) + } + return nil +} + +func getGroupKey(group, id string) ds.Key { + return dsGroupPrefix.ChildString(group).ChildString(id) +} + +func encode(r Request) ([]byte, error) { + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(r); err != nil { + return nil, fmt.Errorf("encoding request: %v", err) + } + return buf.Bytes(), nil +} + +func decode(v []byte) (r Request, err error) { + var buf bytes.Buffer + if _, err := buf.Write(v); err != nil { + return r, fmt.Errorf("writing key value: %v", err) + } + dec := gob.NewDecoder(&buf) + if err := dec.Decode(&r); err != nil { + return r, fmt.Errorf("decoding key value: %v", err) + } + return r, nil +} diff --git a/pinning/queue/queue_test.go b/pinning/queue/queue_test.go new file mode 100644 index 0000000..9367819 --- /dev/null +++ b/pinning/queue/queue_test.go @@ -0,0 +1,475 @@ +package queue_test + +import ( + "context" + "errors" + "io/ioutil" + "strings" + "testing" + "time" + + logging "github.com/ipfs/go-log/v2" + mbase "github.com/multiformats/go-multibase" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + openapi "github.com/textileio/go-buckets/pinning/openapi/go" + . "github.com/textileio/go-buckets/pinning/queue" + "github.com/textileio/go-threads/util" +) + +func init() { + if err := util.SetLogLevels(map[string]logging.LogLevel{ + "buckets/ps-queue": logging.LevelDebug, + }); err != nil { + panic(err) + } +} + +func TestQueue_NewID(t *testing.T) { + t.Parallel() + q := newQueue(t) + + // Ensure monotonic + var last string + for i := 0; i < 10000; i++ { + id, err := q.NewID(time.Now()) + require.NoError(t, err) + + if i > 0 { + assert.Greater(t, id, last) + } + last = id + } +} + +func TestQueue_ListRequests(t *testing.T) { + t.Parallel() + q := newQueue(t) + + t.Run("pagination", func(t *testing.T) { + limit := 100 + now := time.Now() + key := newBucketkey(t) + ids := make([]string, limit) + for i := 0; i < limit; i++ { + now = now.Add(time.Millisecond * 10) + p := newParams(key, now, time.Millisecond, succeed) + r, err := q.AddRequest(p) + require.NoError(t, err) + ids[i] = r.Requestid + } + + time.Sleep(time.Second) // wait for all to finish + + // Listing from another key should return 0 results + l, err := q.ListRequests(newBucketkey(t), Query{Statuses: []openapi.Status{openapi.PINNED}}) + require.NoError(t, err) + assert.Len(t, l, 0) + + // Using before and after should error + l, err = q.ListRequests(key, Query{Before: time.Now(), After: time.Now()}) + require.Error(t, err) + + // Empty query, should return newest 10 records + l, err = q.ListRequests(key, Query{Statuses: []openapi.Status{openapi.PINNED}}) + require.NoError(t, err) + assert.Len(t, l, 10) + assert.Equal(t, ids[limit-1], l[0].Requestid) + assert.Equal(t, ids[limit-10], l[9].Requestid) + + // Get next page, should return next 10 records + before := l[len(l)-1].Created + l, err = q.ListRequests(key, Query{Statuses: []openapi.Status{openapi.PINNED}, Before: before}) + require.NoError(t, err) + assert.Len(t, l, 10) + assert.Equal(t, ids[limit-11], l[0].Requestid) + assert.Equal(t, ids[limit-20], l[9].Requestid) + + // Get previous page, should return the first page in reverse order + after := l[0].Created + l, err = q.ListRequests(key, Query{Statuses: []openapi.Status{openapi.PINNED}, After: after}) + require.NoError(t, err) + assert.Len(t, l, 10) + assert.Equal(t, ids[limit-10], l[0].Requestid) + assert.Equal(t, ids[limit-1], l[9].Requestid) + }) + + t.Run("filter by cids", func(t *testing.T) { + // This is a bit awkward since these tests use the openapi.Pin.Cid field to encode + // how long the mock handler should take and whether or not it should succeed. + // That said, the mechanism is still being tested: Match a string. + + limit := 10 + now := time.Now() + key := newBucketkey(t) + ids := make([]string, limit) + for i := 0; i < limit; i++ { + now = now.Add(time.Second) + p := newParams(key, now, time.Duration(i*1000), succeed) + r, err := q.AddRequest(p) + require.NoError(t, err) + ids[i] = r.Requestid + } + + time.Sleep(time.Second) // wait for all to finish + + l, err := q.ListRequests(key, Query{Cids: []string{ + newOutcome(time.Duration(0), succeed), + }}) + require.NoError(t, err) + assert.Len(t, l, 1) + assert.Equal(t, ids[0], l[0].Requestid) + + l, err = q.ListRequests(key, Query{Cids: []string{ + newOutcome(time.Duration(1000), succeed), + newOutcome(time.Duration(4000), succeed), + newOutcome(time.Duration(7000), succeed), + }}) + require.NoError(t, err) + assert.Len(t, l, 3) + assert.Equal(t, ids[7], l[0].Requestid) + assert.Equal(t, ids[4], l[1].Requestid) + assert.Equal(t, ids[1], l[2].Requestid) + }) + + t.Run("filter by name", func(t *testing.T) { + now := time.Now() + key := newBucketkey(t) + + p := newParams(key, now, time.Millisecond, succeed) + p.Name = "My Pin" + _, err := q.AddRequest(p) + require.NoError(t, err) + + p = newParams(key, now.Add(time.Second), time.Millisecond, succeed) + p.Name = "Your Pin" + _, err = q.AddRequest(p) + require.NoError(t, err) + + time.Sleep(time.Second) // wait for all to finish + + // Case-sensitive exact match + l, err := q.ListRequests(key, Query{Name: "My Pin"}) + require.NoError(t, err) + assert.Len(t, l, 1) + + // Case-insensitive exact match + l, err = q.ListRequests(key, Query{Name: "MY pin", Match: openapi.IEXACT}) + require.NoError(t, err) + assert.Len(t, l, 1) + + // Case-sensitive partial match + l, err = q.ListRequests(key, Query{Name: "Pin", Match: openapi.PARTIAL}) + require.NoError(t, err) + assert.Len(t, l, 2) + + // Case-insensitive partial match + l, err = q.ListRequests(key, Query{Name: "pin", Match: openapi.IPARTIAL}) + require.NoError(t, err) + assert.Len(t, l, 2) + + // No match + l, err = q.ListRequests(key, Query{Name: "Their Pin"}) + require.NoError(t, err) + assert.Len(t, l, 0) + }) + + t.Run("filter by status", func(t *testing.T) { + limit := 100 + now := time.Now() + key := newBucketkey(t) + ids := make([]string, limit) + sids := make([]string, 0) + fids := make([]string, 0) + for i := 0; i < limit; i++ { + now = now.Add(time.Second) + var o outcomeType + if i%2 != 0 { + o = succeed + } else { + o = fail + } + p := newParams(key, now, time.Millisecond, o) + r, err := q.AddRequest(p) + require.NoError(t, err) + if i%2 != 0 { + o = succeed + sids = append(sids, r.Requestid) + } else { + o = fail + fids = append(fids, r.Requestid) + } + ids[i] = r.Requestid + } + + time.Sleep(time.Second) // wait for all to finish + + // List first page of all request statuses, ensure entire order is maintained + l, err := q.ListRequests(key, Query{Statuses: []openapi.Status{openapi.PINNED, openapi.FAILED}}) + require.NoError(t, err) + assert.Len(t, l, 10) + for i := 0; i < len(l); i++ { + assert.Equal(t, ids[limit-(i+1)], l[i].Requestid) + } + + // List only "pinned" statuses + l, err = q.ListRequests(key, Query{Statuses: []openapi.Status{openapi.PINNED}}) + require.NoError(t, err) + assert.Len(t, l, 10) + assert.Equal(t, sids[limit/2-1], l[0].Requestid) + assert.Equal(t, sids[limit/2-10], l[9].Requestid) + + // List only "failed" statuses + l, err = q.ListRequests(key, Query{Statuses: []openapi.Status{openapi.FAILED}}) + require.NoError(t, err) + assert.Len(t, l, 10) + assert.Equal(t, fids[limit/2-1], l[0].Requestid) + assert.Equal(t, fids[limit/2-10], l[9].Requestid) + }) + + t.Run("filter by meta", func(t *testing.T) { + now := time.Now() + key := newBucketkey(t) + + p := newParams(key, now, time.Millisecond, succeed) + p.Meta = map[string]string{ + "app": "angry dogs", + "dog": "eddy", + } + _, err := q.AddRequest(p) + require.NoError(t, err) + + p = newParams(key, now.Add(time.Second), time.Millisecond, succeed) + p.Meta = map[string]string{ + "app": "angry dogs", + "dog": "clyde", + } + _, err = q.AddRequest(p) + require.NoError(t, err) + + time.Sleep(time.Second) // wait for all to finish + + // No keys provided + l, err := q.ListRequests(key, Query{Meta: map[string]string{}}) + require.NoError(t, err) + assert.Len(t, l, 2) + + // Match one key + l, err = q.ListRequests(key, Query{Meta: map[string]string{ + "app": "angry dogs", + }}) + require.NoError(t, err) + assert.Len(t, l, 2) + + // Match two keys + l, err = q.ListRequests(key, Query{Meta: map[string]string{ + "app": "angry dogs", + "dog": "eddy", + }}) + require.NoError(t, err) + assert.Len(t, l, 1) + + // Partial match should not work + l, err = q.ListRequests(key, Query{Meta: map[string]string{ + "app": "angry dogs", + "dog": "biff", + }}) + require.NoError(t, err) + assert.Len(t, l, 0) + + // No key matches + l, err = q.ListRequests(key, Query{Meta: map[string]string{ + "weather": "nice", + }}) + require.NoError(t, err) + assert.Len(t, l, 0) + + // No value matches + l, err = q.ListRequests(key, Query{Meta: map[string]string{ + "app": "angry cats", + }}) + require.NoError(t, err) + assert.Len(t, l, 0) + }) +} + +func TestQueue_AddRequest(t *testing.T) { + t.Parallel() + q := newQueue(t) + + p := newParams(newBucketkey(t), time.Now(), time.Millisecond, succeed) + r, err := q.AddRequest(p) + require.NoError(t, err) + assert.Equal(t, openapi.QUEUED, r.Status) + + // Allow to finish + time.Sleep(time.Millisecond * 10) + + got, err := q.GetRequest(p.Key, r.Requestid) + require.NoError(t, err) + assert.Equal(t, openapi.PINNED, got.Status) +} + +func TestQueue_ReplaceRequest(t *testing.T) { + t.Parallel() + q := newQueue(t) + + p := newParams(newBucketkey(t), time.Now(), time.Millisecond, fail) + r, err := q.AddRequest(p) + require.NoError(t, err) + + newPin := openapi.Pin{ + Cid: newOutcome(time.Millisecond, succeed), + } + + // Request will skip status "queued" and go straight to "pinning" since + // there is no backlog of work. That means we can't replace it until it's + // "pinned" or "failed" + _, err = q.ReplaceRequest(p.Key, r.Requestid, newPin) + require.Error(t, err) + + // Allow to finish + time.Sleep(time.Millisecond * 10) + + got, err := q.GetRequest(p.Key, r.Requestid) + require.NoError(t, err) + assert.Equal(t, openapi.FAILED, got.Status) + + _, err = q.ReplaceRequest(p.Key, r.Requestid, newPin) + require.NoError(t, err) + + // Allow to finish + time.Sleep(time.Millisecond * 10) + + got, err = q.GetRequest(p.Key, r.Requestid) + require.NoError(t, err) + assert.Equal(t, openapi.PINNED, got.Status) +} + +func TestQueue_RemoveRequest(t *testing.T) { + t.Parallel() + q := newQueue(t) + + p := newParams(newBucketkey(t), time.Now(), time.Millisecond, succeed) + r, err := q.AddRequest(p) + require.NoError(t, err) + + // Request will skip status "queued" and go straight to "pinning" since + // there is no backlog of work. That means we can't remove it until it's + // "pinned" or "failed" + err = q.RemoveRequest(p.Key, r.Requestid) + require.Error(t, err) + + // Allow to finish + time.Sleep(time.Millisecond * 10) + + err = q.RemoveRequest(p.Key, r.Requestid) + require.NoError(t, err) + + // Allow to finish + time.Sleep(time.Millisecond * 10) + + _, err = q.GetRequest(p.Key, r.Requestid) + require.Error(t, err) +} + +func TestQueueProcessing(t *testing.T) { + t.Parallel() + q := newQueue(t) + + limit := 500 + now := time.Now() + key1 := newBucketkey(t) + for i := 0; i < limit; i++ { + now = now.Add(time.Second) + var o outcomeType + if i%10 != 0 { + o = succeed + } else { + o = fail + } + p := newParams(key1, now, time.Millisecond*100, o) + _, err := q.AddRequest(p) + require.NoError(t, err) + } + + time.Sleep(time.Second * 5) // wait for all to finish + + l, err := q.ListRequests(key1, Query{ + Statuses: []openapi.Status{openapi.PINNING, openapi.QUEUED}, + Limit: limit, + }) + require.NoError(t, err) + assert.Len(t, l, 0) // zero should be queued + + l, err = q.ListRequests(key1, Query{ + Statuses: []openapi.Status{openapi.PINNED}, + Limit: limit, + }) + require.NoError(t, err) + assert.Len(t, l, 450) // expected amount should be pinned + + l, err = q.ListRequests(key1, Query{ + Statuses: []openapi.Status{openapi.FAILED}, + Limit: limit, + }) + require.NoError(t, err) + assert.Len(t, l, 50) // expected amount should be failed +} + +func newQueue(t *testing.T) *Queue { + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + s, err := util.NewBadgerDatastore(dir, "pinq") + require.NoError(t, err) + q, err := NewQueue(s, handler) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, q.Close()) + require.NoError(t, s.Close()) + }) + return q +} + +func handler(_ context.Context, r Request) error { + d, t := parseOutcome(r.Pin.Cid) + time.Sleep(d) + if t == succeed { + return nil + } else { + return errors.New("bummer") + } +} + +type outcomeType string + +const ( + succeed outcomeType = "success" + fail = "failure" +) + +func newOutcome(d time.Duration, t outcomeType) string { + return strings.Join([]string{d.String(), string(t)}, ",") +} + +func parseOutcome(o string) (time.Duration, outcomeType) { + parts := strings.Split(o, ",") + d, _ := time.ParseDuration(parts[0]) + return d, outcomeType(parts[1]) +} + +func newParams(k string, t time.Time, d time.Duration, o outcomeType) RequestParams { + return RequestParams{ + Pin: openapi.Pin{ + Cid: newOutcome(d, o), + }, + Time: t, + Key: k, + } +} + +func newBucketkey(t *testing.T) string { + k, err := mbase.Encode(mbase.Base36, util.GenerateRandomBytes(20)) + require.NoError(t, err) + return k +} diff --git a/pinning/service.go b/pinning/service.go new file mode 100644 index 0000000..8103498 --- /dev/null +++ b/pinning/service.go @@ -0,0 +1,400 @@ +package pinning + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + c "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + iface "github.com/ipfs/interface-go-ipfs-core" + "github.com/libp2p/go-libp2p-core/peer" + maddr "github.com/multiformats/go-multiaddr" + "github.com/textileio/go-buckets" + openapi "github.com/textileio/go-buckets/pinning/openapi/go" + q "github.com/textileio/go-buckets/pinning/queue" + "github.com/textileio/go-threads/core/did" + core "github.com/textileio/go-threads/core/thread" + kt "github.com/textileio/go-threads/db/keytransform" +) + +var ( + log = logging.Logger("buckets/ps") + + // ErrPermissionDenied indicates an identity does not have the required athorization. + ErrPermissionDenied = errors.New("permission denied") + + // PinTimeout is the max time taken to pin a Cid. + PinTimeout = time.Hour + + // statusTimeout is the timeout used when updating pin status in a bucket. + statusTimeout = time.Minute + + // connectTimeout is the timeout used when connecting to pin origins. + connectTimeout = time.Second * 10 +) + +// GetLocalAddrs returns a list of addresses announced by the ipfs instance. +func GetLocalAddrs(ipfs iface.CoreAPI) ([]maddr.Multiaddr, error) { + key, err := ipfs.Key().Self(context.Background()) + if err != nil { + return nil, err + } + paddr, err := maddr.NewMultiaddr("/p2p/" + key.ID().String()) + if err != nil { + return nil, err + } + addrs, err := ipfs.Swarm().LocalAddrs(context.Background()) + if err != nil { + return nil, err + } + paddrs := make([]maddr.Multiaddr, len(addrs)) + for i, a := range addrs { + paddrs[i] = a.Encapsulate(paddr) + } + return paddrs, nil +} + +// Service provides a bucket-based IPFS Pinning Service based on the OpenAPI spec: +// https://github.com/ipfs/pinning-services-api-spec +type Service struct { + lib *buckets.Buckets + queue *q.Queue + addrs []string +} + +// NewService returns a new pinning Service. +func NewService(lib *buckets.Buckets, store kt.TxnDatastoreExtended) (*Service, error) { + s := &Service{lib: lib} + queue, err := q.NewQueue(store, s.handleRequest) + if err != nil { + return nil, fmt.Errorf("creating queue: %v", err) + } + s.queue = queue + return s, nil +} + +// Close the Service. +func (s *Service) Close() error { + return s.queue.Close() +} + +// ListPins returns a list of openapi.PinStatus matching the Query. +func (s *Service) ListPins( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + query q.Query, +) ([]openapi.PinStatus, error) { + // Ensure bucket is readable by identity + if ok, err := s.lib.IsReadablePath(ctx, thread, key, identity, ""); err != nil { + return nil, fmt.Errorf("authenticating read: %v", err) + } else if !ok { + return nil, ErrPermissionDenied + } + + list, err := s.queue.ListRequests(key, query) + if err != nil { + return nil, fmt.Errorf("listing requests: %v", err) + } + + delegates, err := s.getDelegates() + if err != nil { + return nil, fmt.Errorf("getting delegates: %v", err) + } + for _, r := range list { + r.Delegates = delegates + } + + log.Debugf("listed %d requests in %s", len(list), key) + return list, nil +} + +// AddPin adds an openapi.Pin to a bucket. +func (s *Service) AddPin( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + pin openapi.Pin, +) (*openapi.PinStatus, error) { + // Ensure bucket is writable by identity + if ok, err := s.lib.IsWritablePath(ctx, thread, key, identity, ""); err != nil { + return nil, fmt.Errorf("authenticating write: %v", err) + } else if !ok { + return nil, ErrPermissionDenied + } + + // Verify pin cid + if _, err := c.Decode(pin.Cid); err != nil { + return nil, fmt.Errorf("decoding pin cid: %v", err) + } + + // Enqueue request + r, err := s.queue.AddRequest(q.RequestParams{ + Pin: pin, + Time: time.Now(), + Thread: thread, + Key: key, + Identity: identity, + }) + if err != nil { + return nil, fmt.Errorf("adding request: %v", err) + } + + r.Delegates, err = s.getDelegates() + if err != nil { + return nil, fmt.Errorf("getting delegates: %v", err) + } + + log.Debugf("added request %s in %s", r.Requestid, key) + return r, nil +} + +// handleRequest attempts to pin the request Cid to the bucket path (request ID). +// Note: The openapi.PinStatus held in the bucket won't be set to "pinning" +// because we have that info in the embedded queue and can therefore avoid an extra bucket write. +// In other words, the bucket state will show "queued", "pinned", or "failed", but not "pinning". +func (s *Service) handleRequest(ctx context.Context, r q.Request) error { + log.Debugf("handling request: %s", r.Requestid) + + // Open a transaction that we can use to control blocking since we may need + // to push a failure to the bucket if SetPath fails. + txn, err := s.lib.NewTxn(r.Thread, r.Key, r.Identity) + if err != nil { + return err + } + defer txn.Close() + + fail := func(reason error) error { + if err := s.failRequest(ctx, txn, r, reason); err != nil { + log.Debugf("failing request: %v", err) + } + return reason + } + + cid, err := c.Decode(r.Pin.Cid) + if err != nil { + return fail(fmt.Errorf("decoding cid: %v", err)) + } + + if r.Remove { + // Remove path from the bucket + ctx, cancel := context.WithTimeout(ctx, statusTimeout) + defer cancel() + if _, _, err := txn.RemovePath(ctx, nil, r.Requestid); buckets.IsPathNotFoundErr(err) { + return fmt.Errorf("removing path %s: %v", r.Requestid, err) + } else if err != nil { + return fail(fmt.Errorf("removing path %s: %v", r.Requestid, err)) + } + } else { + pctx, pcancel := context.WithTimeout(ctx, PinTimeout) + defer pcancel() + + // Connect to Pin.Origins + go s.connectOrigins(pctx, r.Pin.Origins) + + // Replace placeholder at path with openapi.Pin.Cid + if _, _, err := txn.SetPath( + pctx, + nil, + r.Requestid, + cid, + map[string]interface{}{ + "pin": map[string]interface{}{ + "status": openapi.PINNED, + }, + }, + ); buckets.IsPathNotFoundErr(err) { + return fmt.Errorf("setting path %s: %v", r.Requestid, err) + } else if err != nil { + return fail(fmt.Errorf("setting path %s: %v", r.Requestid, err)) + } + } + + log.Debugf("request completed: %s", r.Requestid) + return nil +} + +// failRequest updates the bucket path (request ID) with an error. +func (s *Service) failRequest(ctx context.Context, txn *buckets.Txn, r q.Request, reason error) error { + log.Debugf("failing request %s with reason: %v", r.Requestid, reason) + + // Update placeholder with failure reason + ctx, cancel := context.WithTimeout(ctx, statusTimeout) + defer cancel() + if _, err := txn.PushPath( + ctx, + nil, + buckets.PushPathsInput{ + Path: r.Requestid, + Reader: strings.NewReader(fmt.Sprintf("pin %s failed: %v", r.Pin.Cid, reason)), + Meta: map[string]interface{}{ + "pin": map[string]interface{}{ + "status": openapi.FAILED, + }, + }, + }, + ); err != nil { + return fmt.Errorf("pushing status to bucket: %v", err) + } + + log.Debugf("request failed: %s", r.Requestid) + return nil +} + +// GetPin returns an openapi.PinStatus. +func (s *Service) GetPin( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + id string, +) (*openapi.PinStatus, error) { + // Ensure bucket is readable by identity + if ok, err := s.lib.IsReadablePath(ctx, thread, key, identity, ""); err != nil { + return nil, fmt.Errorf("authenticating read: %v", err) + } else if !ok { + return nil, ErrPermissionDenied + } + + r, err := s.queue.GetRequest(key, id) + if err != nil { + return nil, fmt.Errorf("getting request: %w", err) + } + + r.Delegates, err = s.getDelegates() + if err != nil { + return nil, fmt.Errorf("getting delegates: %v", err) + } + + log.Debugf("got request %s in %s", id, key) + return r, nil +} + +// ReplacePin replaces an openapi.PinStatus with another. +func (s *Service) ReplacePin( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + id string, + pin openapi.Pin, +) (*openapi.PinStatus, error) { + // Ensure bucket is writable by identity + if ok, err := s.lib.IsWritablePath(ctx, thread, key, identity, ""); err != nil { + return nil, fmt.Errorf("authenticating write: %v", err) + } else if !ok { + return nil, ErrPermissionDenied + } + + // Verify pin cid + if _, err := c.Decode(pin.Cid); err != nil { + return nil, fmt.Errorf("decoding pin cid: %v", err) + } + + r, err := s.queue.ReplaceRequest(key, id, pin) + if err != nil { + return nil, fmt.Errorf("replacing request: %w", err) + } + + r.Delegates, err = s.getDelegates() + if err != nil { + return nil, fmt.Errorf("getting delegates: %v", err) + } + + log.Debugf("replaced request %s in %s", id, key) + return r, nil +} + +// RemovePin removes an openapi.PinStatus from a bucket. +func (s *Service) RemovePin( + ctx context.Context, + thread core.ID, + key string, + identity did.Token, + id string, +) error { + // Ensure bucket is writable by identity + if ok, err := s.lib.IsWritablePath(ctx, thread, key, identity, ""); err != nil { + return fmt.Errorf("authenticating write: %v", err) + } else if !ok { + return ErrPermissionDenied + } + + if err := s.queue.RemoveRequest(key, id); err != nil { + return fmt.Errorf("removing request: %w", err) + } + + log.Debugf("removed request %s in %s", id, key) + return nil +} + +func (s *Service) connectOrigins(ctx context.Context, origins []string) { + ctx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + + var wg sync.WaitGroup + for _, o := range origins { + wg.Add(1) + go func(o string) { + defer wg.Done() + pinfo, err := peerInfoFromOrigin(o) + if err != nil { + log.Errorf("error %v", err) + return + } + if err := s.lib.Ipfs().Swarm().Connect(ctx, *pinfo); err != nil { + log.Debugf("error connecting to %s: %v", pinfo.ID, err) + } else { + log.Debugf("connected to %s", pinfo.ID) + } + }(o) + } + wg.Wait() +} + +func peerInfoFromOrigin(origin string) (*peer.AddrInfo, error) { + addr, err := maddr.NewMultiaddr(origin) + if err != nil { + return nil, fmt.Errorf("decoding origin: %v", err) + } + parts := maddr.Split(addr) + if len(parts) == 0 { + return nil, errors.New("origin has zero components") + } + var p2p maddr.Multiaddr + p2p, parts = parts[len(parts)-1], parts[:len(parts)-1] + p2pv, err := p2p.ValueForProtocol(maddr.P_P2P) + if err != nil { + return nil, fmt.Errorf("origin missing p2p component: %v", err) + } + pid, err := peer.Decode(p2pv) + if err != nil { + return nil, fmt.Errorf("getting peer id: %v", err) + } + return &peer.AddrInfo{ + ID: pid, + Addrs: []maddr.Multiaddr{ + maddr.Join(parts...), + }, + }, nil +} + +func (s *Service) getDelegates() ([]string, error) { + if len(s.addrs) != 0 { + return s.addrs, nil + } + addrs, err := GetLocalAddrs(s.lib.Ipfs()) + if err != nil { + return nil, fmt.Errorf("getting ipfs local addrs: %v", err) + } + for _, a := range addrs { + s.addrs = append(s.addrs, a.String()) + } + return s.addrs, nil +} diff --git a/pull.go b/pull.go index 6b44cf0..b634e4a 100644 --- a/pull.go +++ b/pull.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/interface-go-ipfs-core/path" "github.com/textileio/dcrypto" "github.com/textileio/go-buckets/dag" - "github.com/textileio/go-buckets/util" "github.com/textileio/go-threads/core/did" core "github.com/textileio/go-threads/core/thread" ) @@ -33,14 +32,19 @@ func (r *pathReader) Close() error { return nil } +// PullPath returns a reader to a bucket path. func (b *Buckets) PullPath( ctx context.Context, thread core.ID, - key, pth string, + key string, identity did.Token, + pth string, ) (io.ReadCloser, error) { + if err := thread.Validate(); err != nil { + return nil, fmt.Errorf("invalid thread id: %v", err) + } pth = trimSlash(pth) - instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity) + instance, bpth, err := b.getBucketAndPath(ctx, thread, key, identity, pth) if err != nil { return nil, err } @@ -51,7 +55,7 @@ func (b *Buckets) PullPath( var filePath path.Resolved if instance.IsPrivate() { - buckPath, err := util.NewResolvedPath(instance.Path) + buckPath, err := dag.NewResolvedPath(instance.Path) if err != nil { return nil, err } @@ -102,6 +106,7 @@ func (b *Buckets) PullPath( return r, nil } +// PullIPFSPath returns a reader to an IPFS path. func (b *Buckets) PullIPFSPath(ctx context.Context, pth string) (io.ReadCloser, error) { node, err := b.ipfs.Unixfs().Get(ctx, path.New(pth)) if err != nil { diff --git a/push.go b/push.go index 05c7925..193cfca 100644 --- a/push.go +++ b/push.go @@ -16,16 +16,23 @@ import ( "github.com/textileio/dcrypto" "github.com/textileio/go-buckets/collection" "github.com/textileio/go-buckets/dag" - "github.com/textileio/go-buckets/util" "github.com/textileio/go-threads/core/did" core "github.com/textileio/go-threads/core/thread" ) -type PushPathsChunk struct { +// PushPathsInput is used to with PushPath and PushPaths to specify a reader or chunked data to push to a path. +type PushPathsInput struct { + // Path is a bucket relative path at which to insert data. Path string - Data []byte + // Reader from which to write data into Path. + Reader io.Reader + // Chunk should be used to add chunked data when a plain io.Reader is not available. + Chunk []byte + // Meta is optional metadata the will be persisted under Path. + Meta map[string]interface{} } +// PushPathsResult contains the result of a single push result from PuthPath or PushPaths. type PushPathsResult struct { Path string Cid c.Cid @@ -34,29 +41,96 @@ type PushPathsResult struct { Bucket *Bucket } -func (b *Buckets) PushPaths( +// PushPath pushes data to a single bucket path. +func (b *Buckets) PushPath( ctx context.Context, thread core.ID, key string, + identity did.Token, + root path.Resolved, + input PushPathsInput, +) (*PushPathsResult, error) { + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + return nil, err + } + defer txn.Close() + return txn.PushPath(ctx, root, input) +} + +// PushPath is Txn based PushPath. +func (t *Txn) PushPath( + ctx context.Context, root path.Resolved, + input PushPathsInput, +) (*PushPathsResult, error) { + in, out, errs := t.PushPaths(ctx, root) + if len(errs) != 0 { + err := <-errs + return nil, err + } + + go func() { + in <- input + close(in) + }() + + result := &PushPathsResult{} + for { + select { + case res := <-out: + result = &res + case err := <-errs: + return result, err + } + } +} + +// PushPaths pushes data to one or more bucket paths. +// The returned channels are used to push one or more PushPathsInput. +// Each input will result in a PushPathsResult. +// Use this method to bulk write data to a bucket. +func (b *Buckets) PushPaths( + ctx context.Context, + thread core.ID, + key string, identity did.Token, -) (chan<- PushPathsChunk, <-chan PushPathsResult, <-chan error) { - lk := b.locks.Get(lock(key)) - lk.Acquire() + root path.Resolved, +) (chan<- PushPathsInput, <-chan PushPathsResult, <-chan error) { + errs := make(chan error, 1) + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + errs <- err + return nil, nil, errs + } + + in, out, perrs := txn.PushPaths(ctx, root) + go func() { + defer txn.Close() + for err := range perrs { + errs <- err + return + } + }() + return in, out, errs +} - in := make(chan PushPathsChunk) +// PushPaths is Txn based PushPaths. +func (t *Txn) PushPaths( + ctx context.Context, + root path.Resolved, +) (chan<- PushPathsInput, <-chan PushPathsResult, <-chan error) { + in := make(chan PushPathsInput) out := make(chan PushPathsResult) errs := make(chan error, 1) - instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) + instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) if err != nil { errs <- err - lk.Release() return in, out, errs } if root != nil && root.String() != instance.Path { errs <- ErrNonFastForward - lk.Release() return in, out, errs } readOnlyInstance := instance.Copy() @@ -71,13 +145,13 @@ func (b *Buckets) PushPaths( queue := newFileQueue() for { select { - case chunk, ok := <-in: + case input, ok := <-in: if !ok { wg.Wait() // Request ended normally, wait for pending jobs close(doneCh) return } - pth, err := parsePath(chunk.Path) + pth, err := parsePath(input.Path) if err != nil { errCh <- fmt.Errorf("parsing path: %v", err) return @@ -85,14 +159,20 @@ func (b *Buckets) PushPaths( ctxLock.RLock() ctx := ctx ctxLock.RUnlock() - fa, err := queue.add(ctx, b.ipfs.Unixfs(), pth, func() ([]byte, error) { + fa, err := queue.add(ctx, t.b.ipfs.Unixfs(), pth, input.Meta, func() ([]byte, error) { wg.Add(1) readOnlyInstance.UpdatedAt = time.Now().UnixNano() readOnlyInstance.SetMetadataAtPath(pth, collection.Metadata{ UpdatedAt: readOnlyInstance.UpdatedAt, + Info: input.Meta, }) readOnlyInstance.UnsetMetadataWithPrefix(pth + "/") - if err := b.c.Verify(ctx, thread, readOnlyInstance, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Verify( + ctx, + t.thread, + readOnlyInstance, + collection.WithIdentity(t.identity), + ); err != nil { return nil, fmt.Errorf("verifying bucket update: %v", err) } key, err := readOnlyInstance.GetFileEncryptionKeyForPath(pth) @@ -106,8 +186,17 @@ func (b *Buckets) PushPaths( return } - if len(chunk.Data) > 0 { - if _, err := fa.writer.Write(chunk.Data); err != nil { + if input.Reader != nil { + if _, err := io.Copy(fa.writer, input.Reader); err != nil { + errCh <- fmt.Errorf("piping reader: %v", err) + return + } + if err := fa.writer.Close(); err != nil { + errCh <- fmt.Errorf("closing writer: %v", err) + return + } + } else if len(input.Chunk) > 0 { + if _, err := fa.writer.Write(input.Chunk); err != nil { errCh <- fmt.Errorf("writing chunk: %v", err) return } @@ -122,13 +211,13 @@ func (b *Buckets) PushPaths( }() var changed bool - sctx := util.NewClonedContext(ctx) + sctx := newClonedContext(ctx) saveWithErr := func(err error) error { cancel() if !changed { return err } - if serr := b.saveAndPublish(sctx, thread, instance, identity); serr != nil { + if serr := t.b.saveAndPublish(sctx, t.thread, t.identity, instance); serr != nil { if err != nil { return err } @@ -140,7 +229,6 @@ func (b *Buckets) PushPaths( } go func() { - defer lk.Release() for { select { case res := <-addedCh: @@ -148,7 +236,7 @@ func (b *Buckets) PushPaths( ctx2 := ctx ctxLock.RUnlock() - fn, err := b.ipfs.ResolveNode(ctx2, res.resolved) + fn, err := t.b.ipfs.ResolveNode(ctx2, res.resolved) if err != nil { errs <- saveWithErr(fmt.Errorf("resolving added node: %v", err)) return @@ -158,7 +246,7 @@ func (b *Buckets) PushPaths( if instance.IsPrivate() { ctx2, dir, err = dag.InsertNodeAtPath( ctx2, - b.ipfs, + t.b.ipfs, fn, path.Join(path.New(instance.Path), res.path), instance.GetLinkEncryptionKey(), @@ -168,7 +256,7 @@ func (b *Buckets) PushPaths( return } } else { - dir, err = b.ipfs.Object().AddLink( + dir, err = t.b.ipfs.Object().AddLink( ctx2, path.New(instance.Path), res.path, @@ -179,7 +267,7 @@ func (b *Buckets) PushPaths( errs <- saveWithErr(fmt.Errorf("adding bucket link: %v", err)) return } - ctx2, err = dag.UpdateOrAddPin(ctx2, b.ipfs, path.New(instance.Path), dir) + ctx2, err = dag.UpdateOrAddPin(ctx2, t.b.ipfs, path.New(instance.Path), dir) if err != nil { errs <- saveWithErr(fmt.Errorf("updating bucket pin: %v", err)) return @@ -189,6 +277,7 @@ func (b *Buckets) PushPaths( instance.UpdatedAt = time.Now().UnixNano() instance.SetMetadataAtPath(res.path, collection.Metadata{ UpdatedAt: instance.UpdatedAt, + Info: res.meta, }) instance.UnsetMetadataWithPrefix(res.path + "/") @@ -197,7 +286,7 @@ func (b *Buckets) PushPaths( Cid: res.resolved.Cid(), Size: res.size, Pinned: dag.GetPinnedBytes(ctx2), - Bucket: instanceToBucket(thread, instance), + Bucket: instanceToBucket(t.thread, instance), } ctxLock.Lock() @@ -230,6 +319,7 @@ type addedFile struct { path string resolved path.Resolved size int64 + meta map[string]interface{} } type fileQueue struct { @@ -245,6 +335,7 @@ func (q *fileQueue) add( ctx context.Context, ufs iface.UnixfsAPI, pth string, + meta map[string]interface{}, addFunc func() ([]byte, error), doneCh chan<- addedFile, errCh chan<- error, @@ -314,8 +405,24 @@ func (q *fileQueue) add( errCh <- fmt.Errorf("getting file size: %v", err) return } - doneCh <- addedFile{path: pth, resolved: res, size: int64(added)} + doneCh <- addedFile{ + path: pth, + resolved: res, + size: int64(added), + meta: meta, + } }() return fa, nil } + +// newClonedContext returns a context with the same Values but not inherited cancelation. +func newClonedContext(ctx context.Context) context.Context { + return valueOnlyContext{Context: ctx} +} + +type valueOnlyContext struct{ context.Context } + +func (valueOnlyContext) Deadline() (deadline time.Time, ok bool) { return } +func (valueOnlyContext) Done() <-chan struct{} { return nil } +func (valueOnlyContext) Err() error { return nil } diff --git a/remove.go b/remove.go index 40a34f0..d0de93b 100644 --- a/remove.go +++ b/remove.go @@ -11,47 +11,55 @@ import ( core "github.com/textileio/go-threads/core/thread" ) +// RemovePath removed a path from a bucket. +// All children under the path will be removed and unpinned. func (b *Buckets) RemovePath( ctx context.Context, thread core.ID, - key, pth string, - root path.Resolved, + key string, identity did.Token, + root path.Resolved, + pth string, ) (int64, *Bucket, error) { - lk := b.locks.Get(lock(key)) - lk.Acquire() - defer lk.Release() + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + return 0, nil, err + } + defer txn.Close() + return txn.RemovePath(ctx, root, pth) +} +// RemovePath is Txn based RemovePath. +func (t *Txn) RemovePath(ctx context.Context, root path.Resolved, pth string) (int64, *Bucket, error) { pth, err := parsePath(pth) if err != nil { return 0, nil, err } - instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) + instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) if err != nil { return 0, nil, err } - if root != nil && root.String() != instance.Path { return 0, nil, ErrNonFastForward } instance.UpdatedAt = time.Now().UnixNano() instance.UnsetMetadataWithPrefix(pth) - if err := b.c.Verify(ctx, thread, instance, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { return 0, nil, err } - ctx, dirPath, err := b.removePath(ctx, instance, pth) + ctx, dirPath, err := t.b.removePath(ctx, instance, pth) if err != nil { return 0, nil, err } instance.Path = dirPath.String() - if err := b.saveAndPublish(ctx, thread, instance, identity); err != nil { + if err := t.b.saveAndPublish(ctx, t.thread, t.identity, instance); err != nil { return 0, nil, err } - log.Debugf("removed %s from %s", pth, key) - return dag.GetPinnedBytes(ctx), instanceToBucket(thread, instance), nil + log.Debugf("removed %s from %s", pth, t.key) + return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil } diff --git a/set.go b/set.go index b9592f8..9434532 100644 --- a/set.go +++ b/set.go @@ -12,30 +12,50 @@ import ( core "github.com/textileio/go-threads/core/thread" ) +// SetPath pulls data from IPFS into a bucket path. func (b *Buckets) SetPath( ctx context.Context, thread core.ID, - key, pth string, - cid c.Cid, + key string, identity did.Token, + root path.Resolved, + pth string, + cid c.Cid, + meta map[string]interface{}, ) (int64, *Bucket, error) { - lck := b.locks.Get(lock(key)) - lck.Acquire() - defer lck.Release() + txn, err := b.NewTxn(thread, key, identity) + if err != nil { + return 0, nil, err + } + defer txn.Close() + return txn.SetPath(ctx, root, pth, cid, meta) +} - instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity)) +// SetPath is Txn based SetPath. +func (t *Txn) SetPath( + ctx context.Context, + root path.Resolved, + pth string, + cid c.Cid, + meta map[string]interface{}, +) (int64, *Bucket, error) { + instance, err := t.b.c.GetSafe(ctx, t.thread, t.key, collection.WithIdentity(t.identity)) if err != nil { return 0, nil, err } + if root != nil && root.String() != instance.Path { + return 0, nil, ErrNonFastForward + } pth = trimSlash(pth) instance.UpdatedAt = time.Now().UnixNano() instance.SetMetadataAtPath(pth, collection.Metadata{ UpdatedAt: instance.UpdatedAt, + Info: meta, }) instance.UnsetMetadataWithPrefix(pth + "/") - if err := b.c.Verify(ctx, thread, instance, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { return 0, nil, err } @@ -50,15 +70,15 @@ func (b *Buckets) SetPath( } buckPath := path.New(instance.Path) - ctx, dirPath, err := b.setPathFromExistingCid(ctx, instance, buckPath, pth, cid, linkKey, fileKey) + ctx, dirPath, err := t.b.setPathFromExistingCid(ctx, instance, buckPath, pth, cid, linkKey, fileKey) if err != nil { return 0, nil, err } instance.Path = dirPath.String() - if err := b.c.Save(ctx, thread, instance, collection.WithIdentity(identity)); err != nil { + if err := t.b.c.Save(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil { return 0, nil, err } log.Debugf("set %s to %s", pth, cid) - return dag.GetPinnedBytes(ctx), instanceToBucket(thread, instance), nil + return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil } diff --git a/util/ctxutil.go b/util/ctxutil.go deleted file mode 100644 index 05ad061..0000000 --- a/util/ctxutil.go +++ /dev/null @@ -1,18 +0,0 @@ -package util - -import ( - "context" - "time" -) - -// NewClonedContext returns a context with the same Values -// but not inherited cancelation. -func NewClonedContext(ctx context.Context) context.Context { - return valueOnlyContext{Context: ctx} -} - -type valueOnlyContext struct{ context.Context } - -func (valueOnlyContext) Deadline() (deadline time.Time, ok bool) { return } -func (valueOnlyContext) Done() <-chan struct{} { return nil } -func (valueOnlyContext) Err() error { return nil } diff --git a/util/util.go b/util/util.go deleted file mode 100644 index 9899399..0000000 --- a/util/util.go +++ /dev/null @@ -1,79 +0,0 @@ -package util - -import ( - "crypto/rand" - "fmt" - "strings" - - "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core/path" - maddr "github.com/multiformats/go-multiaddr" - mbase "github.com/multiformats/go-multibase" -) - -func GenerateRandomBytes(n int) []byte { - b := make([]byte, n) - _, err := rand.Read(b) - if err != nil { - panic(err) - } - return b -} - -func MakeToken(n int) string { - bytes := GenerateRandomBytes(n) - encoded, err := mbase.Encode(mbase.Base32, bytes) - if err != nil { - panic(err) - } - return encoded -} - -func MustParseAddr(str string) maddr.Multiaddr { - addr, err := maddr.NewMultiaddr(str) - if err != nil { - panic(err) - } - return addr -} - -func NewResolvedPath(s string) (path.Resolved, error) { - parts := strings.SplitN(s, "/", 3) - if len(parts) != 3 { - return nil, fmt.Errorf("path is not resolvable") - } - c, err := cid.Decode(parts[2]) - if err != nil { - return nil, err - } - return path.IpfsPath(c), nil -} - -func ParsePath(p path.Path) (resolved path.Resolved, fpath string, err error) { - parts := strings.SplitN(p.String(), "/", 4) - if len(parts) < 3 { - err = fmt.Errorf("path does not contain a resolvable segment") - return - } - c, err := cid.Decode(parts[2]) - if err != nil { - return - } - if len(parts) > 3 { - fpath = parts[3] - } - return path.IpfsPath(c), fpath, nil -} - -func ByteCountDecimal(b int64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) -}