wip?

Author: hacdias

examples/gateway/proxy/main_test.go

@@ -9,9 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/ipfs/boxo/blockservice"
 	"github.com/ipfs/boxo/examples/gateway/common"
-	"github.com/ipfs/boxo/exchange/offline"
 	"github.com/ipfs/boxo/gateway"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/stretchr/testify/assert"
@@ -25,10 +23,10 @@ const (
 func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server {
 	blockStore, err := gateway.NewProxyBlockstore([]string{rs.URL}, nil)
 	require.NoError(t, err)
-	blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
+	// blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
 	routing := newProxyRouting(rs.URL, nil)
 
-	backend, err := gateway.NewBlocksBackend(blockService, gateway.WithValueStore(routing))
+	backend, err := gateway.NewGraphGatewayBackend(blockStore, gateway.WithValueStore(routing))
 	require.NoError(t, err)
 
 	handler := common.NewHandler(backend)

gateway/backend_remote.go

@@ -9,15 +9,23 @@ import (
 	"sync"
 	"time"
 
+	bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
 	"github.com/ipfs/boxo/verifcid"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-unixfsnode"
+	"github.com/ipfs/go-unixfsnode/data"
 	"github.com/ipld/go-car"
+	dagpb "github.com/ipld/go-codec-dagpb"
 	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/datamodel"
 	"github.com/ipld/go-ipld-prime/linking"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+	"github.com/ipld/go-ipld-prime/schema"
+	"github.com/ipld/go-ipld-prime/traversal"
+	"github.com/ipld/go-ipld-prime/traversal/selector"
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 	"github.com/multiformats/go-multihash"
 )
 
@@ -135,3 +143,169 @@ func getLinksystem(fn getBlock) *ipld.LinkSystem {
 	unixfsnode.AddUnixFSReificationToLinkSystem(&lsys)
 	return &lsys
 }
+
+// walkGatewaySimpleSelector2 walks the subgraph described by the path and terminal element parameters
+func walkGatewaySimpleSelector2(ctx context.Context, terminalBlk blocks.Block, dagScope DagScope, entityRange *DagByteRange, lsys *ipld.LinkSystem) error {
+	lctx := ipld.LinkContext{Ctx: ctx}
+	var err error
+
+	// If the scope is the block, we only need the root block of the last element of the path, which we have.
+	if dagScope == DagScopeBlock {
+		return nil
+	}
+
+	// decode the terminal block into a node
+	pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+		if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
+			return tlnkNd.LinkTargetNodePrototype(), nil
+		}
+		return basicnode.Prototype.Any, nil
+	})
+
+	pathTerminalCidLink := cidlink.Link{Cid: terminalBlk.Cid()}
+	np, err := pc(pathTerminalCidLink, lctx)
+	if err != nil {
+		return err
+	}
+
+	decoder, err := lsys.DecoderChooser(pathTerminalCidLink)
+	if err != nil {
+		return err
+	}
+	nb := np.NewBuilder()
+	blockData := terminalBlk.RawData()
+	if err := decoder(nb, bytes.NewReader(blockData)); err != nil {
+		return err
+	}
+	lastCidNode := nb.Build()
+
+	// TODO: Evaluate:
+	// Does it matter that we're ignoring the "remainder" portion of the traversal in GetCAR?
+	// Does it matter that we're using a linksystem with the UnixFS reifier for dagscope=all?
+
+	// If we're asking for everything then give it
+	if dagScope == DagScopeAll {
+		sel, err := selector.ParseSelector(selectorparse.CommonSelector_ExploreAllRecursively)
+		if err != nil {
+			return err
+		}
+
+		progress := traversal.Progress{
+			Cfg: &traversal.Config{
+				Ctx:                            ctx,
+				LinkSystem:                     *lsys,
+				LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser,
+				LinkVisitOnlyOnce:              false, // Despite being safe for the "all" selector we do this walk anyway since this is how we will be receiving the blocks
+			},
+		}
+
+		if err := progress.WalkMatching(lastCidNode, sel, func(progress traversal.Progress, node datamodel.Node) error {
+			return nil
+		}); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// From now on, dag-scope=entity!
+	// Since we need more of the graph load it to figure out what we have
+	// This includes determining if the terminal node is UnixFS or not
+	if pbn, ok := lastCidNode.(dagpb.PBNode); !ok {
+		// If it's not valid dag-pb then we're done
+		return nil
+	} else if !pbn.FieldData().Exists() {
+		// If it's not valid UnixFS then we're done
+		return nil
+	} else if unixfsFieldData, decodeErr := data.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil {
+		// If it's not valid dag-pb and UnixFS then we're done
+		return nil
+	} else {
+		switch unixfsFieldData.FieldDataType().Int() {
+		case data.Data_Directory, data.Data_Symlink:
+			// These types are non-recursive so we're done
+			return nil
+		case data.Data_Raw, data.Data_Metadata:
+			// TODO: for now, we decided to return nil here. The different implementations are inconsistent
+			// and UnixFS is not properly specified: https://github.com/ipfs/specs/issues/316.
+			// 		- Is Data_Raw different from Data_File?
+			//		- Data_Metadata is handled differently in boxo/ipld/unixfs and go-unixfsnode.
+			return nil
+		case data.Data_HAMTShard:
+			// Return all elements in the map
+			_, err := lsys.KnownReifiers["unixfs-preload"](lctx, lastCidNode, lsys)
+			if err != nil {
+				return err
+			}
+			return nil
+		case data.Data_File:
+			nd, err := unixfsnode.Reify(lctx, lastCidNode, lsys)
+			if err != nil {
+				return err
+			}
+
+			fnd, ok := nd.(datamodel.LargeBytesNode)
+			if !ok {
+				return fmt.Errorf("could not process file since it did not present as large bytes")
+			}
+			f, err := fnd.AsLargeBytes()
+			if err != nil {
+				return err
+			}
+
+			// Get the entity range. If it's empty, assume the defaults (whole file).
+			effectiveRange := entityRange
+			if effectiveRange == nil {
+				effectiveRange = &DagByteRange{
+					From: 0,
+				}
+			}
+
+			from := effectiveRange.From
+
+			// If we're starting to read based on the end of the file, find out where that is.
+			var fileLength int64
+			foundFileLength := false
+			if effectiveRange.From < 0 {
+				fileLength, err = f.Seek(0, io.SeekEnd)
+				if err != nil {
+					return err
+				}
+				from = fileLength + effectiveRange.From
+				foundFileLength = true
+			}
+
+			// If we're reading until the end of the file then do it
+			if effectiveRange.To == nil {
+				if _, err := f.Seek(from, io.SeekStart); err != nil {
+					return err
+				}
+				_, err = io.Copy(io.Discard, f)
+				return err
+			}
+
+			to := *effectiveRange.To
+			if (*effectiveRange.To) < 0 && !foundFileLength {
+				fileLength, err = f.Seek(0, io.SeekEnd)
+				if err != nil {
+					return err
+				}
+				to = fileLength + *effectiveRange.To
+				foundFileLength = true
+			}
+
+			numToRead := 1 + to - from
+			if numToRead < 0 {
+				return fmt.Errorf("tried to read less than zero bytes")
+			}
+
+			if _, err := f.Seek(from, io.SeekStart); err != nil {
+				return err
+			}
+			_, err = io.CopyN(io.Discard, f, numToRead)
+			return err
+		default:
+			// Not a supported type, so we're done
+			return nil
+		}
+	}
+}

gateway/backend_remote_2.go

@@ -1108,7 +1108,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, p path.ImmutablePath, param
 			l := getLinksystem(teeBlock)
 
 			// First resolve the path since we always need to.
-			_, terminalCid, remainder, _, err := resolvePathWithRootsAndBlock(ctx, p, l)
+			_, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(ctx, p, l)
 			if err != nil {
 				return err
 			}
@@ -1133,7 +1133,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, p path.ImmutablePath, param
 				blockBuffer = nil
 			}
 
-			err = walkGatewaySimpleSelector(ctx, terminalCid, remainder, params, l)
+			err = walkGatewaySimpleSelector2(ctx, terminalBlk, params.Scope, params.Range, l)
 			if err != nil {
 				return err
 			}