Bucket Pinning Service

.bingo/.gitignore

@@ -5,7 +5,6 @@
 # But not these files:
 !.gitignore
 !*.mod
-!*.sum
 !README.md
 !Variables.mk
 !variables.env

.bingo/Variables.mk

@@ -20,35 +20,35 @@ BUF := $(GOBIN)/buf-v0.20.5
 $(BUF): .bingo/buf.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
 	@echo "(re)installing $(GOBIN)/buf-v0.20.5"
-	@cd .bingo && $(GO) build -modfile=buf.mod -o=$(GOBIN)/buf-v0.20.5 "github.com/bufbuild/buf/cmd/buf"
+	@cd .bingo && $(GO) build -mod=mod -modfile=buf.mod -o=$(GOBIN)/buf-v0.20.5 "github.com/bufbuild/buf/cmd/buf"
 
 GOMPLATE := $(GOBIN)/gomplate-v3.8.0
 $(GOMPLATE): .bingo/gomplate.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
 	@echo "(re)installing $(GOBIN)/gomplate-v3.8.0"
-	@cd .bingo && $(GO) build -modfile=gomplate.mod -o=$(GOBIN)/gomplate-v3.8.0 "github.com/hairyhenderson/gomplate/v3/cmd/gomplate"
+	@cd .bingo && $(GO) build -mod=mod -modfile=gomplate.mod -o=$(GOBIN)/gomplate-v3.8.0 "github.com/hairyhenderson/gomplate/v3/cmd/gomplate"
 
 GOVVV := $(GOBIN)/govvv-v0.3.0
 $(GOVVV): .bingo/govvv.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
 	@echo "(re)installing $(GOBIN)/govvv-v0.3.0"
-	@cd .bingo && $(GO) build -modfile=govvv.mod -o=$(GOBIN)/govvv-v0.3.0 "github.com/ahmetb/govvv"
+	@cd .bingo && $(GO) build -mod=mod -modfile=govvv.mod -o=$(GOBIN)/govvv-v0.3.0 "github.com/ahmetb/govvv"
 
 GOX := $(GOBIN)/gox-v1.0.1
 $(GOX): .bingo/gox.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
 	@echo "(re)installing $(GOBIN)/gox-v1.0.1"
-	@cd .bingo && $(GO) build -modfile=gox.mod -o=$(GOBIN)/gox-v1.0.1 "github.com/mitchellh/gox"
+	@cd .bingo && $(GO) build -mod=mod -modfile=gox.mod -o=$(GOBIN)/gox-v1.0.1 "github.com/mitchellh/gox"
 
 PROTOC_GEN_BUF_CHECK_BREAKING := $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5
 $(PROTOC_GEN_BUF_CHECK_BREAKING): .bingo/protoc-gen-buf-check-breaking.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
 	@echo "(re)installing $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5"
-	@cd .bingo && $(GO) build -modfile=protoc-gen-buf-check-breaking.mod -o=$(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-breaking"
+	@cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-breaking.mod -o=$(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-breaking"
 
 PROTOC_GEN_BUF_CHECK_LINT := $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5
 $(PROTOC_GEN_BUF_CHECK_LINT): .bingo/protoc-gen-buf-check-lint.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
 	@echo "(re)installing $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5"
-	@cd .bingo && $(GO) build -modfile=protoc-gen-buf-check-lint.mod -o=$(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-lint"
+	@cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-lint.mod -o=$(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-lint"
 

.bingo/buf.mod

@@ -1,5 +1,5 @@
 module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
-go 1.16
+go 1.14
 
 require github.com/bufbuild/buf v0.20.5 // cmd/buf

.bingo/go.mod

@@ -1,3 +1,3 @@
 module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files.
 
-go 1.16
+go 1.14

.bingo/gomplate.mod

@@ -1,5 +1,5 @@
 module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
-go 1.16
+go 1.14
 
 require github.com/hairyhenderson/gomplate/v3 v3.8.0 // cmd/gomplate

.bingo/govvv.mod

@@ -1,5 +1,5 @@
 module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
-go 1.16
+go 1.14
 
 require github.com/ahmetb/govvv v0.3.0

.bingo/gox.mod

@@ -1,5 +1,5 @@
 module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
-go 1.16
+go 1.14
 
 require github.com/mitchellh/gox v1.0.1

.bingo/protoc-gen-buf-check-breaking.mod

@@ -1,5 +1,5 @@
 module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
-go 1.16
+go 1.14
 
 require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-breaking

.bingo/protoc-gen-buf-check-lint.mod

@@ -1,5 +1,5 @@
 module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
 
-go 1.16
+go 1.14
 
 require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-lint

.github/workflows/build.yml

@@ -24,23 +24,17 @@ jobs:
   buck:
     name: Buck CLI
     runs-on: ubuntu-latest
+    container: golang:1.16.0-buster
     steps:
-      - name: setup
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.16
       - name: checkout
         uses: actions/checkout@v1
       - name: build
         run: make build-buck
   buckd:
     name: Buck Daemon
     runs-on: ubuntu-latest
+    container: golang:1.16.0-buster
     steps:
-      - name: setup
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.16
       - name: checkout
         uses: actions/checkout@v1
       - name: build

.github/workflows/release.yml

@@ -6,11 +6,8 @@ jobs:
   release-platform-builds:
     name: Release Builds
     runs-on: ubuntu-latest
+    container: golang:1.16.0-buster
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.16
       - name: Check out code
         uses: actions/checkout@v1
       - name: Cache dependencies

README.md

@@ -10,6 +10,8 @@
 
 Join us on our [public Slack channel](https://slack.textile.io/) for news, discussions, and status updates. [Check out our blog](https://medium.com/textileio) for the latest posts and announcements.
 
+### WARNING: This repo is pointing to a [feature branch](https://github.com/textileio/go-threads/pull/498) of `go-threads` that handles identities as Decentralized Identifiers (DIDs) and should be considered alpha. The [textile repo](https://github.com/textileio/textile) contains the non-DID-based bucket implementation currently compatible with [Textile's public hub](https://cloud.hub.textile.io/#/access). DID-based buckets will be integrated into the hub mid 2021. In the meantime, you can still use this repo for standalone bucket peers.
+
 ## Table of Contents
 
 - [Security](#security)

access.go

@@ -14,28 +14,48 @@ import (
 	core "github.com/textileio/go-threads/core/thread"
 )
 
+// PushPathAccessRoles pushes new access roles to bucket paths.
+// Access roles are keyed by did.DID.
+// Roles are inherited by path children.
 func (b *Buckets) PushPathAccessRoles(
 	ctx context.Context,
 	thread core.ID,
-	key, pth string,
-	roles map[did.DID]collection.Role,
+	key string,
 	identity did.Token,
+	root path.Resolved,
+	pth string,
+	roles map[did.DID]collection.Role,
 ) (int64, *Bucket, error) {
-	lk := b.locks.Get(lock(key))
-	lk.Acquire()
-	defer lk.Release()
+	txn, err := b.NewTxn(thread, key, identity)
+	if err != nil {
+		return 0, nil, err
+	}
+	defer txn.Close()
+	return txn.PushPathAccessRoles(ctx, root, pth, roles)
+}
 
+// PushPathAccessRoles is Txn based PushPathInfo.
+func (t *Txn) PushPathAccessRoles(
+	ctx context.Context,
+	root path.Resolved,
+	pth string,
+	roles map[did.DID]collection.Role,
+) (int64, *Bucket, error) {
 	pth, err := parsePath(pth)
 	if err != nil {
 		return 0, nil, err
 	}
 
-	instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity)
+	instance, bpth, err := t.b.getBucketAndPath(ctx, t.thread, t.key, t.identity, pth)
 	if err != nil {
 		return 0, nil, err
 	}
+	if root != nil && root.String() != instance.Path {
+		return 0, nil, ErrNonFastForward
+	}
+
 	linkKey := instance.GetLinkEncryptionKey()
-	pathNode, err := dag.GetNodeAtPath(ctx, b.ipfs, bpth, linkKey)
+	pathNode, err := dag.GetNodeAtPath(ctx, t.b.ipfs, bpth, linkKey)
 	if err != nil {
 		return 0, nil, err
 	}
@@ -86,7 +106,7 @@ func (b *Buckets) PushPathAccessRoles(
 				return 0, nil, err
 			}
 		}
-		if err := b.c.Verify(ctx, thread, instance, collection.WithIdentity(identity)); err != nil {
+		if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil {
 			return 0, nil, err
 		}
 
@@ -97,7 +117,7 @@ func (b *Buckets) PushPathAccessRoles(
 			}
 			nmap, err := dag.EncryptDag(
 				ctx,
-				b.ipfs,
+				t.b.ipfs,
 				pathNode,
 				pth,
 				linkKey,
@@ -117,41 +137,46 @@ func (b *Buckets) PushPathAccessRoles(
 			}
 			pn := nmap[pathNode.Cid()].Node
 			var dirPath path.Resolved
-			ctx, dirPath, err = dag.InsertNodeAtPath(ctx, b.ipfs, pn, path.Join(path.New(instance.Path), pth), linkKey)
+			ctx, dirPath, err = dag.InsertNodeAtPath(ctx, t.b.ipfs, pn, path.Join(path.New(instance.Path), pth), linkKey)
 			if err != nil {
 				return 0, nil, err
 			}
-			ctx, err = dag.AddAndPinNodes(ctx, b.ipfs, nodes)
+			ctx, err = dag.AddAndPinNodes(ctx, t.b.ipfs, nodes)
 			if err != nil {
 				return 0, nil, err
 			}
 			instance.Path = dirPath.String()
 		}
 
-		if err := b.c.Save(ctx, thread, instance, collection.WithIdentity(identity)); err != nil {
+		if err := t.b.c.Save(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil {
 			return 0, nil, err
 		}
 	}
 
-	log.Debugf("pushed access roles for %s in %s", pth, key)
-	return dag.GetPinnedBytes(ctx), instanceToBucket(thread, instance), nil
+	log.Debugf("pushed access roles for %s in %s", pth, t.key)
+	return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil
 }
 
+// PullPathAccessRoles pulls access roles for a bucket path.
 func (b *Buckets) PullPathAccessRoles(
 	ctx context.Context,
 	thread core.ID,
-	key, pth string,
+	key string,
 	identity did.Token,
+	pth string,
 ) (map[did.DID]collection.Role, error) {
+	if err := thread.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid thread id: %v", err)
+	}
 	pth, err := parsePath(pth)
 	if err != nil {
 		return nil, err
 	}
-	instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity)
+	instance, bpth, err := b.getBucketAndPath(ctx, thread, key, identity, pth)
 	if err != nil {
 		return nil, err
 	}
-	if _, err := dag.GetNodeAtPath(ctx, b.ipfs, bpth, instance.GetLinkEncryptionKey()); err != nil {
+	if _, err = dag.GetNodeAtPath(ctx, b.ipfs, bpth, instance.GetLinkEncryptionKey()); err != nil {
 		return nil, fmt.Errorf("could not resolve path: %s", pth)
 	}
 	md, _, ok := instance.GetMetadataForPath(pth, false)
@@ -162,3 +187,55 @@ func (b *Buckets) PullPathAccessRoles(
 	log.Debugf("pulled access roles for %s in %s", pth, key)
 	return md.Roles, nil
 }
+
+// IsReadablePath returns whether or not a path is readable by an identity.
+func (b *Buckets) IsReadablePath(
+	ctx context.Context,
+	thread core.ID,
+	key string,
+	identity did.Token,
+	pth string,
+) (bool, error) {
+	if err := thread.Validate(); err != nil {
+		return false, fmt.Errorf("invalid thread id: %v", err)
+	}
+	pth, err := parsePath(pth)
+	if err != nil {
+		return false, err
+	}
+	instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity))
+	if err != nil {
+		return false, err
+	}
+	_, doc, err := core.Validate(identity, nil)
+	if err != nil {
+		return false, err
+	}
+	return instance.IsReadablePath(pth, doc.ID), nil
+}
+
+// IsWritablePath returns whether or not a path is writable by an identity.
+func (b *Buckets) IsWritablePath(
+	ctx context.Context,
+	thread core.ID,
+	key string,
+	identity did.Token,
+	pth string,
+) (bool, error) {
+	if err := thread.Validate(); err != nil {
+		return false, fmt.Errorf("invalid thread id: %v", err)
+	}
+	pth, err := parsePath(pth)
+	if err != nil {
+		return false, err
+	}
+	instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity))
+	if err != nil {
+		return false, err
+	}
+	_, doc, err := core.Validate(identity, nil)
+	if err != nil {
+		return false, err
+	}
+	return instance.IsWritablePath(pth, doc.ID), nil
+}

api/apitest/apitest.go

@@ -19,7 +19,6 @@ import (
 	"github.com/textileio/go-buckets"
 	"github.com/textileio/go-buckets/api/common"
 	"github.com/textileio/go-buckets/ipns"
-	"github.com/textileio/go-buckets/util"
 	dbc "github.com/textileio/go-threads/api/client"
 	"github.com/textileio/go-threads/core/did"
 	tdb "github.com/textileio/go-threads/db"
@@ -30,25 +29,26 @@ import (
 func NewService(t *testing.T) (listenAddr string, host did.DID) {
 	err := tutil.SetLogLevels(map[string]logging.LogLevel{
 		"buckets":      logging.LevelDebug,
-		"buckets-api":  logging.LevelDebug,
-		"buckets-ipns": logging.LevelDebug,
-		"buckets-dns":  logging.LevelDebug,
+		"buckets/api":  logging.LevelDebug,
+		"buckets/ipns": logging.LevelDebug,
+		"buckets/dns":  logging.LevelDebug,
 	})
 	require.NoError(t, err)
 
 	threadsAddr := GetThreadsApiAddr()
 	net, err := nc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...)
 	require.NoError(t, err)
 
-	// @todo: Fix me
+	// @todo: Use service description to build client
 	doc, err := net.GetServices(context.Background())
 	require.NoError(t, err)
 
 	db, err := dbc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...)
 	require.NoError(t, err)
 	ipfs, err := httpapi.NewApi(GetIPFSApiMultiAddr())
 	require.NoError(t, err)
-	ipnsm, err := ipns.NewManager(tdb.NewTxMapDatastore(), ipfs)
+	ipnsms := tdb.NewTxMapDatastore()
+	ipnsm, err := ipns.NewManager(ipnsms, ipfs)
 	require.NoError(t, err)
 	lib, err := buckets.NewBuckets(net, db, ipfs, ipnsm, nil)
 	require.NoError(t, err)
@@ -64,6 +64,8 @@ func NewService(t *testing.T) (listenAddr string, host did.DID) {
 		server.Stop()
 		require.NoError(t, lib.Close())
 		require.NoError(t, ipnsm.Close())
+		require.NoError(t, ipnsms.Close())
+		require.NoError(t, db.Close())
 		require.NoError(t, net.Close())
 	})
 
@@ -83,9 +85,9 @@ func GetThreadsApiAddr() string {
 func GetIPFSApiMultiAddr() ma.Multiaddr {
 	env := os.Getenv("IPFS_API_MULTIADDR")
 	if env != "" {
-		return util.MustParseAddr(env)
+		return tutil.MustParseAddr(env)
 	}
-	return util.MustParseAddr("/ip4/127.0.0.1/tcp/5012")
+	return tutil.MustParseAddr("/ip4/127.0.0.1/tcp/5012")
 }
 
 // StartServices starts an ipfs and threads node for tests.